Instruction:
Implement a function in Python 3 to
get a dictionary of running Spark applications.

Response:
def _yarn_get_running_spark_apps(self, rm_address, requests_config, tags):
        """
        Return a dictionary of {app_id: (app_name, tracking_url)} for running Spark applications.

        The `app_id` returned is that of the YARN application. This will eventually be mapped into
        a Spark application ID.
        """
        metrics_json = self._rest_request_to_json(
            rm_address,
            YARN_APPS_PATH,
            YARN_SERVICE_CHECK,
            requests_config,
            tags,
            states=APPLICATION_STATES,
            applicationTypes=YARN_APPLICATION_TYPES,
        )

        running_apps = {}

        if metrics_json.get('apps'):
            if metrics_json['apps'].get('app') is not None:

                for app_json in metrics_json['apps']['app']:
                    app_id = app_json.get('id')
                    tracking_url = app_json.get('trackingUrl')
                    app_name = app_json.get('name')

                    if app_id and tracking_url and app_name:
                        running_apps[app_id] = (app_name, tracking_url)

        return running_apps


Instruction:
Write a Python 3 script for
getting a dictionary of Spark application IDs and tracking URLs for each Spark application.

Response:
def _get_spark_app_ids(self, running_apps, requests_config, tags):
        """
        Traverses the Spark application master in YARN to get a Spark application ID.

        Return a dictionary of {app_id: (app_name, tracking_url)} for Spark applications
        """
        spark_apps = {}
        for app_id, (app_name, tracking_url) in iteritems(running_apps):
            response = self._rest_request_to_json(
                tracking_url, SPARK_APPS_PATH, SPARK_SERVICE_CHECK, requests_config, tags
            )

            for app in response:
                app_id = app.get('id')
                app_name = app.get('name')

                if app_id and app_name:
                    spark_apps[app_id] = (app_name, tracking_url)

        return spark_apps


Instruction:
Create a Python 3 function to
get metrics for each Spark job.

Response:
def _spark_job_metrics(self, instance, running_apps, addl_tags, requests_config):
        """
        Get metrics for each Spark job.
        """
        for app_id, (app_name, tracking_url) in iteritems(running_apps):

            base_url = self._get_request_url(instance, tracking_url)
            response = self._rest_request_to_json(
                base_url, SPARK_APPS_PATH, SPARK_SERVICE_CHECK, requests_config, addl_tags, app_id, 'jobs'
            )

            for job in response:

                status = job.get('status')

                tags = ['app_name:%s' % str(app_name)]
                tags.extend(addl_tags)
                tags.append('status:%s' % str(status).lower())

                self._set_metrics_from_json(tags, job, SPARK_JOB_METRICS)
                self._set_metric('spark.job.count', COUNT, 1, tags)


Instruction:
Create a Python 3 function for
getting metrics for each Spark executor.

Response:
def _spark_executor_metrics(self, instance, running_apps, addl_tags, requests_config):
        """
        Get metrics for each Spark executor.
        """
        for app_id, (app_name, tracking_url) in iteritems(running_apps):

            base_url = self._get_request_url(instance, tracking_url)
            response = self._rest_request_to_json(
                base_url, SPARK_APPS_PATH, SPARK_SERVICE_CHECK, requests_config, addl_tags, app_id, 'executors'
            )

            tags = ['app_name:%s' % str(app_name)]
            tags.extend(addl_tags)

            for executor in response:
                if executor.get('id') == 'driver':
                    self._set_metrics_from_json(tags, executor, SPARK_DRIVER_METRICS)
                else:
                    self._set_metrics_from_json(tags, executor, SPARK_EXECUTOR_METRICS)

            if len(response):
                self._set_metric('spark.executor.count', COUNT, len(response), tags)


Instruction:
Can you write a function in Python 3 where it
gets metrics for each Spark RDD.

Response:
def _spark_rdd_metrics(self, instance, running_apps, addl_tags, requests_config):
        """
        Get metrics for each Spark RDD.
        """
        for app_id, (app_name, tracking_url) in iteritems(running_apps):

            base_url = self._get_request_url(instance, tracking_url)
            response = self._rest_request_to_json(
                base_url, SPARK_APPS_PATH, SPARK_SERVICE_CHECK, requests_config, addl_tags, app_id, 'storage/rdd'
            )

            tags = ['app_name:%s' % str(app_name)]
            tags.extend(addl_tags)

            for rdd in response:
                self._set_metrics_from_json(tags, rdd, SPARK_RDD_METRICS)

            if len(response):
                self._set_metric('spark.rdd.count', COUNT, len(response), tags)


Instruction:
Implement a Python 3 function for
getting metrics for each application streaming statistics.

Response:
def _spark_streaming_statistics_metrics(self, instance, running_apps, addl_tags, requests_config):
        """
        Get metrics for each application streaming statistics.
        """
        for app_id, (app_name, tracking_url) in iteritems(running_apps):
            try:
                base_url = self._get_request_url(instance, tracking_url)
                response = self._rest_request_to_json(
                    base_url,
                    SPARK_APPS_PATH,
                    SPARK_SERVICE_CHECK,
                    requests_config,
                    addl_tags,
                    app_id,
                    'streaming/statistics',
                )
                self.log.debug('streaming/statistics: %s', response)
                tags = ['app_name:%s' % str(app_name)]
                tags.extend(addl_tags)

                # NOTE: response is a dict
                self._set_metrics_from_json(tags, response, SPARK_STREAMING_STATISTICS_METRICS)
            except HTTPError as e:
                # NOTE: If api call returns response 404
                # then it means that the application is not a streaming application, we should skip metric submission
                if e.response.status_code != 404:
                    raise


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _set_metric(self, metric_name, metric_type, value, tags=None):
        if tags is None:
            tags = []
        if metric_type == GAUGE:
            self.gauge(metric_name, value, tags=tags)
        elif metric_type == COUNT:
            self.count(metric_name, value, tags=tags)
        elif metric_type == MONOTONIC_COUNT:
            self.monotonic_count(metric_name, value, tags=tags)
        else:
            self.log.error('Metric type "{}" unknown'.format(metric_type))

Response: Set a metric in the log.


Instruction:
How would you code a function in Python 3 to
perform a request to Spark and return the response.

Response:
def _rest_request(self, url, object_path, service_name, requests_config, tags, *args, **kwargs):
        """
        Query the given URL and return the response
        """
        service_check_tags = ['url:%s' % self._get_url_base(url)] + tags

        if object_path:
            url = self._join_url_dir(url, object_path)

        # Load SSL configuration, if available.
        # ssl_verify can be a bool or a string
        # (http://docs.python-requests.org/en/latest/user/advanced/#ssl-cert-verification)
        if isinstance(requests_config.ssl_verify, bool) or isinstance(requests_config.ssl_verify, str):
            verify = requests_config.ssl_verify
        else:
            verify = None
        if requests_config.ssl_cert and requests_config.ssl_key:
            cert = (requests_config.ssl_cert, requests_config.ssl_key)
        elif requests_config.ssl_cert:
            cert = requests_config.ssl_cert
        else:
            cert = None

        # Add args to the url
        if args:
            for directory in args:
                url = self._join_url_dir(url, directory)

        # Add kwargs as arguments
        if kwargs:
            query = '&'.join(['{0}={1}'.format(key, value) for key, value in iteritems(kwargs)])
            url = urljoin(url, '?' + query)

        old_keytab_path = None
        if requests_config.kerberos_keytab:
            old_keytab_path = os.getenv('KRB5_CLIENT_KTNAME')
            os.environ['KRB5_CLIENT_KTNAME'] = requests_config.kerberos_keytab

        try:
            self.log.debug('Spark check URL: %s' % url)
            response = requests.get(url, auth=requests_config.auth, verify=verify, cert=cert)

            response.raise_for_status()

        except Timeout as e:
            self.service_check(
                service_name,
                AgentCheck.CRITICAL,
                tags=service_check_tags,
                message='Request timeout: {0}, {1}'.format(url, e),
            )
            raise

        except (HTTPError, InvalidURL, ConnectionError) as e:
            self.service_check(
                service_name,
                AgentCheck.CRITICAL,
                tags=service_check_tags,
                message='Request failed: {0}, {1}'.format(url, e),
            )
            raise

        except ValueError as e:
            self.service_check(service_name, AgentCheck.CRITICAL, tags=service_check_tags, message=str(e))
            raise

        else:
            return response

        finally:
            if old_keytab_path is not None:
                os.environ['KRB5_CLIENT_KTNAME'] = old_keytab_path


Instruction:
Can you implement a function in Python 3 that
queries the given URL and return the JSON response

Response:
def _rest_request_to_json(self, address, object_path, service_name, requests_config, tags, *args, **kwargs):
        """
        Query the given URL and return the JSON response
        """
        response = self._rest_request(address, object_path, service_name, requests_config, tags, *args, **kwargs)

        try:
            response_json = response.json()

        except JSONDecodeError as e:
            self.service_check(
                service_name,
                AgentCheck.CRITICAL,
                tags=['url:%s' % self._get_url_base(address)] + tags,
                message='JSON Parse failed: {0}'.format(e),
            )
            raise

        return response_json


Instruction:
Can you create a Python 3 function that
returns the base of a URL.

Response:
def _get_url_base(cls, url):
        """
        Return the base of a URL
        """
        s = urlsplit(url)
        return urlunsplit([s.scheme, s.netloc, '', '', ''])


Instruction:
Can you tell what is the following Python 3 function doing
def get_event_tags_from_dn(dn):
    tags = []
    node = get_node_from_dn(dn)
    if node:
        tags.append("node:" + node)
    app = get_app_from_dn(dn)
    if app:
        tags.append("app:" + app)
    bd = get_bd_from_dn(dn)
    if bd:
        tags.append("bd:" + bd)
    cep = get_cep_from_dn(dn)
    if cep:
        tags.append("mac:" + cep)
    ip = get_ip_from_dn(dn)
    if ip:
        tags.append("ip:" + ip)
    epg = get_epg_from_dn(dn)
    if epg:
        tags.append("epg:" + epg)
    return tags

Response: Returns a list of event tags from a DN.


Instruction:
Can you write a function in Python 3 where it
checks if a metric value is zero

Response:
def check_metric_can_be_zero(metric_name, metric_value, json_attributes):
    """
    When a counter is reset, don't send a zero because it will look bad on the graphs
    This checks if the zero makes sense or not
    """
    if "last" in metric_name.lower():
        return True
    if not metric_value:
        return False
    try:
        if metric_value == 0 or metric_value == "0" or metric_value == "0.000000" or float(metric_value) == 0.0:
            if not json_attributes or not json_attributes.get('cnt'):
                return False
            if json_attributes.get('cnt') == "0" or json_attributes.get('cnt') == 0:
                return False
    except ValueError:
        return False
    return True


Instruction:
How would you explain what the following Python 3 function does
def create(ctx, name, integration_type, location, non_interactive, quiet, dry_run):
    repo_choice = ctx.obj['repo_choice']
    root = resolve_path(location) if location else get_root()
    path_sep = os.path.sep

    integration_dir = os.path.join(root, normalize_package_name(name))
    if os.path.exists(integration_dir):
        abort('Path `{}` already exists!'.format(integration_dir))

    template_fields = {}
    if repo_choice != 'core' and not non_interactive and not dry_run:
        template_fields['author'] = click.prompt('Your name')
        template_fields['email'] = click.prompt('Your email')
        template_fields['email_packages'] = template_fields['email']
        click.echo()

    config = construct_template_fields(name, repo_choice, **template_fields)

    files = create_template_files(integration_type, root, config, read=not dry_run)
    file_paths = [file.file_path.replace('{}{}'.format(root, path_sep), '', 1) for file in files]

    path_tree = tree()
    for file_path in file_paths:
        branch = path_tree

        for part in file_path.split(path_sep):
            branch = branch[part]

    if dry_run:
        if quiet:
            echo_info('Will create `{}`'.format(integration_dir))
        else:
            echo_info('Will create in `{}`:'.format(root))
            display_path_tree(path_tree)
        return

    for file in files:
        file.write()

    if quiet:
        echo_info('Created `{}`'.format(integration_dir))
    else:
        echo_info('Created in `{}`:'.format(root))
        display_path_tree(path_tree)

Response: Create a new scaffolding for a new integration.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def freeze(o):
    if isinstance(o, (tuple, list)):
        return tuple(sorted(freeze(e) for e in o))

    if isinstance(o, dict):
        return tuple(sorted((k, freeze(v)) for k, v in iteritems(o)))

    if isinstance(o, (set, frozenset)):
        return tuple(sorted(freeze(e) for e in o))

    return o

Response: Freezes any mutable object including dictionaries and lists for hashing.


Instruction:
Write a Python 3 script to
convert a decimal. Decimal to BID.

Response:
def _decimal_to_128(value):
    """Converts a decimal.Decimal to BID (high bits, low bits).

    :Parameters:
      - `value`: An instance of decimal.Decimal
    """
    with decimal.localcontext(_DEC128_CTX) as ctx:
        value = ctx.create_decimal(value)

    if value.is_infinite():
        return _NINF if value.is_signed() else _PINF

    sign, digits, exponent = value.as_tuple()

    if value.is_nan():
        if digits:
            raise ValueError("NaN with debug payload is not supported")
        if value.is_snan():
            return _NSNAN if value.is_signed() else _PSNAN
        return _NNAN if value.is_signed() else _PNAN

    significand = int("".join([str(digit) for digit in digits]))
    bit_length = _bit_length(significand)

    high = 0
    low = 0
    for i in range(min(64, bit_length)):
        if significand & (1 << i):
            low |= 1 << i

    for i in range(64, bit_length):
        if significand & (1 << i):
            high |= 1 << (i - 64)

    biased_exponent = exponent + _EXPONENT_BIAS

    if high >> 49 == 1:
        high = high & 0x7fffffffffff
        high |= _EXPONENT_MASK
        high |= (biased_exponent & 0x3fff) << 47
    else:
        high |= biased_exponent << 49

    if sign:
        high |= _SIGN

    return high, low


Instruction:
Can you write a function in Python 3 where it
returns an instance of decimal. Decimal for thisCOOKIE.

Response:
def to_decimal(self):
        """Returns an instance of :class:`decimal.Decimal` for this
        :class:`Decimal128`.
        """
        high = self.__high
        low = self.__low
        sign = 1 if (high & _SIGN) else 0

        if (high & _SNAN) == _SNAN:
            return decimal.Decimal((sign, (), 'N'))
        elif (high & _NAN) == _NAN:
            return decimal.Decimal((sign, (), 'n'))
        elif (high & _INF) == _INF:
            return decimal.Decimal((sign, (), 'F'))

        if (high & _EXPONENT_MASK) == _EXPONENT_MASK:
            exponent = ((high & 0x1fffe00000000000) >> 47) - _EXPONENT_BIAS
            return decimal.Decimal((sign, (0,), exponent))
        else:
            exponent = ((high & 0x7fff800000000000) >> 49) - _EXPONENT_BIAS

        arr = bytearray(15)
        mask = 0x00000000000000ff
        for i in range(14, 6, -1):
            arr[i] = (low & mask) >> ((14 - i) << 3)
            mask = mask << 8

        mask = 0x00000000000000ff
        for i in range(6, 0, -1):
            arr[i] = (high & mask) >> ((6 - i) << 3)
            mask = mask << 8

        mask = 0x0001000000000000
        arr[0] = (high & mask) >> 48

        # Have to convert bytearray to bytes for python 2.6.
        # cdecimal only accepts a tuple for digits.
        digits = tuple(
            int(digit) for digit in str(_from_bytes(bytes(arr), 'big')))

        with decimal.localcontext(_DEC128_CTX) as ctx:
            return ctx.create_decimal((sign, digits, exponent))


Instruction:
Create a Python 3 function for
creating an instance of Decimal128 from binary integer value.

Response:
def from_bid(cls, value):
        """Create an instance of :class:`Decimal128` from Binary Integer
        Decimal string.

        :Parameters:
          - `value`: 16 byte string (128-bit IEEE 754-2008 decimal floating
            point in Binary Integer Decimal (BID) format).
        """
        if not isinstance(value, bytes):
            raise TypeError("value must be an instance of bytes")
        if len(value) != 16:
            raise ValueError("value must be exactly 16 bytes")
        return cls((_UNPACK_64(value[8:])[0], _UNPACK_64(value[:8])[0]))


Instruction:
Can you generate the documentation for the following Python 3 function
def _compile_tag_re(self):
        queue_tag_list = []
        for regex_str, tags in iteritems(self._queue_tag_re):
            try:
                queue_tag_list.append([re.compile(regex_str), [t.strip() for t in tags.split(',')]])
            except TypeError:
                log.warning('{} is not a valid regular expression and will be ignored'.format(regex_str))
        return queue_tag_list

Response: Compile regex strings from queue_tag_re option and return list of compiled regex / tag pairs


Instruction:
Write a Python 3 function for
determining the server type from an ismaster response.

Response:
def _get_server_type(doc):
    """Determine the server type from an ismaster response."""
    if not doc.get('ok'):
        return SERVER_TYPE.Unknown

    if doc.get('isreplicaset'):
        return SERVER_TYPE.RSGhost
    elif doc.get('setName'):
        if doc.get('hidden'):
            return SERVER_TYPE.RSOther
        elif doc.get('ismaster'):
            return SERVER_TYPE.RSPrimary
        elif doc.get('secondary'):
            return SERVER_TYPE.RSSecondary
        elif doc.get('arbiterOnly'):
            return SERVER_TYPE.RSArbiter
        else:
            return SERVER_TYPE.RSOther
    elif doc.get('msg') == 'isdbgrid':
        return SERVER_TYPE.Mongos
    else:
        return SERVER_TYPE.Standalone


Instruction:
Explain what the following Python 3 code does
def all_hosts(self):
        return set(imap(common.clean_node, itertools.chain(
            self._doc.get('hosts', []),
            self._doc.get('passives', []),
            self._doc.get('arbiters', []))))

Response: List of hosts passives and arbiters known to this server.


Instruction:
Explain what the following Python 3 code does
def _create_gitlab_runner_prometheus_instance(self, instance, init_config):
        # Mapping from Prometheus metrics names to Datadog ones
        # For now it's a 1:1 mapping
        allowed_metrics = init_config.get('allowed_metrics')
        if allowed_metrics is None:
            raise CheckException("At least one metric must be whitelisted in `allowed_metrics`.")

        gitlab_runner_instance = deepcopy(instance)

        # gitlab_runner uses 'prometheus_endpoint' and not 'prometheus_url', so we have to rename the key
        gitlab_runner_instance['prometheus_url'] = instance.get('prometheus_endpoint', None)

        gitlab_runner_instance.update(
            {
                'namespace': 'gitlab_runner',
                'metrics': allowed_metrics,
                # Defaults that were set when gitlab_runner was based on PrometheusCheck
                'send_monotonic_counter': instance.get('send_monotonic_counter', False),
                'health_service_check': instance.get('health_service_check', False),
            }
        )

        return gitlab_runner_instance

Response: Create a gitlab_runner instance for the Prometheus check.


Instruction:
Make a summary of the following Python 3 code
def _get_jmx_data(self, instance, jmx_address, tags):
        response = self._rest_request_to_json(
            instance, jmx_address, self.JMX_PATH, {'qry': self.HDFS_DATANODE_BEAN_NAME}, tags=tags
        )
        beans = response.get('beans', [])
        return beans

Response: Get namenode beans data from JMX endpoint


Instruction:
How would you code a function in Python 3 to
process HDFS Datanode metrics from given beans and tags

Response:
def _hdfs_datanode_metrics(self, beans, tags):
        """
        Process HDFS Datanode metrics from given beans
        """
        # Only get the first bean
        bean = next(iter(beans))
        bean_name = bean.get('name')

        self.log.debug("Bean name retrieved: {}".format(bean_name))

        for metric, (metric_name, metric_type) in iteritems(self.HDFS_METRICS):
            metric_value = bean.get(metric)
            if metric_value is not None:
                self._set_metric(metric_name, metric_type, metric_value, tags)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def requirements(ctx):
    echo_info('Freezing check releases')
    checks = get_valid_checks()
    checks.remove('datadog_checks_dev')

    entries = []
    for check in checks:
        if check in AGENT_V5_ONLY:
            echo_info('Check `{}` is only shipped with Agent 5, skipping'.format(check))
            continue

        try:
            version = get_version_string(check)
            entries.append('{}\n'.format(get_agent_requirement_line(check, version)))
        except Exception as e:
            echo_failure('Error generating line: {}'.format(e))
            continue

    lines = sorted(entries)

    req_file = get_agent_release_requirements()
    write_file_lines(req_file, lines)
    echo_success('Successfully wrote to `{}`!'.format(req_file))

Response:
Write the requirements - agent - release. txt file at the root of the repo
   .


Instruction:
Given the following Python 3 function, write the documentation
def _nest_payload(self, keys, payload):
        if len(keys) == 0:
            return payload

        return {keys[0]: self._nest_payload(keys[1:], payload)}

Response: Nest a payload in a dict under the keys contained in keys


Instruction:
Can you implement a function in Python 3 that
returns a list of tags from integrations - core representing an Agent release sorted by recent first.

Response:
def get_agent_tags(since, to):
    """
    Return a list of tags from integrations-core representing an Agent release,
    sorted by more recent first.
    """
    agent_tags = sorted(parse_version_info(t) for t in git_tag_list(r'^\d+\.\d+\.\d+$'))

    # default value for `to` is the latest tag
    if to:
        to = parse_version_info(to)
    else:
        to = agent_tags[-1]

    since = parse_version_info(since)

    # filter out versions according to the interval [since, to]
    agent_tags = [t for t in agent_tags if since <= t <= to]

    # reverse so we have descendant order
    return [str(t) for t in reversed(agent_tags)]


Instruction:
Can you generate the documentation for the following Python 3 function
def run(ctx, args):
    if not args:
        click.echo(ctx.get_help())
        return

    with chdir(get_root()):
        result = run_command(args)

    ctx.exit(result.code)

Response: Run commands in the proper repo.


Instruction:
Create a Python 3 function to
normalize the server URL.

Response:
def _normalize_server_url(self, server):
        """
        Check if the server URL starts with a HTTP or HTTPS scheme, fall back to http if not present
        """
        server = server if server.startswith(("http://", "https://")) else "http://{}".format(server)
        return server


Instruction:
Can you generate the documentation for the following Python 3 function
def _get_es_version(self, config):
        try:
            data = self._get_data(config.url, config, send_sc=False)
            # pre-release versions of elasticearch are suffixed with -rcX etc..
            # peel that off so that the map below doesn't error out
            version = data['version']['number'].split('-')[0]
            version = [int(p) for p in version.split('.')[0:3]]
        except AuthenticationError:
            raise
        except Exception as e:
            self.warning("Error while trying to get Elasticsearch version from %s %s" % (config.url, str(e)))
            version = [1, 0, 0]

        self.service_metadata('version', version)
        self.log.debug("Elasticsearch version is %s" % version)
        return version

Response: Get the running version of elasticsearch.


Instruction:
Write a Python 3 function that can
join base url with url.

Response:
def _join_url(self, base, url, admin_forwarder=False):
        """
        overrides `urlparse.urljoin` since it removes base url path
        https://docs.python.org/2/library/urlparse.html#urlparse.urljoin
        """
        if admin_forwarder:
            return base + url
        else:
            return urljoin(base, url)


Instruction:
Explain what the following Python 3 code does
def _get_urls(self, version, cluster_stats):
        pshard_stats_url = "/_stats"
        health_url = "/_cluster/health"

        if version >= [0, 90, 10]:
            pending_tasks_url = "/_cluster/pending_tasks"
            stats_url = "/_nodes/stats" if cluster_stats else "/_nodes/_local/stats"
            if version < [5, 0, 0]:
                # version 5 errors out if the `all` parameter is set
                stats_url += "?all=true"
        else:
            # legacy
            pending_tasks_url = None
            stats_url = "/_cluster/nodes/stats?all=true" if cluster_stats else "/_cluster/nodes/_local/stats?all=true"

        return health_url, stats_url, pshard_stats_url, pending_tasks_url

Response: Compute the URLs we need to hit depending on the running ES version.


Instruction:
Here you have a function in Python 3, explain what it does
def _get_data(self, url, config, send_sc=True):
        # Load basic authentication configuration, if available.
        if config.username and config.password:
            auth = (config.username, config.password)
        else:
            auth = None

        # Load SSL configuration, if available.
        # ssl_verify can be a bool or a string
        # (http://docs.python-requests.org/en/latest/user/advanced/#ssl-cert-verification)
        if isinstance(config.ssl_verify, bool) or isinstance(config.ssl_verify, str):
            verify = config.ssl_verify
        else:
            verify = None

        if config.ssl_cert:
            if config.ssl_key:
                cert = (config.ssl_cert, config.ssl_key)
            else:
                cert = config.ssl_cert
        else:
            cert = None

        resp = None
        try:
            resp = requests.get(
                url, timeout=config.timeout, headers=headers(self.agentConfig), auth=auth, verify=verify, cert=cert
            )
            resp.raise_for_status()
        except Exception as e:
            # this means we've hit a particular kind of auth error that means the config is broken
            if resp and resp.status_code == 400:
                raise AuthenticationError("The ElasticSearch credentials are incorrect")

            if send_sc:
                self.service_check(
                    self.SERVICE_CHECK_CONNECT_NAME,
                    AgentCheck.CRITICAL,
                    message="Error {} when hitting {}".format(e, url),
                    tags=config.service_check_tags,
                )
            raise

        self.log.debug("request to url {} returned: {}".format(url, resp))

        return resp.json()

Response: Get data from an ElasticSearch URL.


Instruction:
Create a Python 3 function to
process a metric from the data dictionary.

Response:
def _process_metric(self, data, metric, xtype, path, xform=None, tags=None, hostname=None):
        """
        data: dictionary containing all the stats
        metric: datadog metric
        path: corresponding path in data, flattened, e.g. thread_pool.bulk.queue
        xform: a lambda to apply to the numerical value
        """
        value = data

        # Traverse the nested dictionaries
        for key in path.split('.'):
            if value is not None:
                value = value.get(key)
            else:
                break

        if value is not None:
            if xform:
                value = xform(value)
            if xtype == "gauge":
                self.gauge(metric, value, tags=tags, hostname=hostname)
            else:
                self.rate(metric, value, tags=tags, hostname=hostname)
        else:
            self.log.debug("Metric not found: %s -> %s", path, metric)


Instruction:
Can you create a Python 3 function that
resets the state and uid set.

Response:
def reset(self):
        """
        Resets state and uid set. To be called asap to free memory
        """
        self.reached_limit = False
        self.count = 0
        self.seen.clear()


Instruction:
Can you generate the documentation for the following Python 3 function
def is_reached(self, uid=None):
        if self.reached_limit:
            return True

        if uid:
            if uid in self.seen:
                return False
            self.count += 1
            self.seen.add(uid)
        else:
            self.count += 1

        if self.count > self.limit:
            if self.warning:
                self.warning(
                    "Check {} exceeded limit of {} {}, ignoring next ones".format(
                        self.check_name, self.limit, self.name
                    )
                )
            self.reached_limit = True
            return True
        return False

Response: This method is called when the count of the object is exceeded the limit.


Instruction:
Write a Python 3 function for
getting the highwater offsets for each topic and partition.

Response:
def _get_broker_offsets(self, instance, topics):
        """
        Fetch highwater offsets for each topic/partition from Kafka cluster.

        Do this for all partitions in the cluster because even if it has no
        consumers, we may want to measure whether producers are successfully
        producing. No need to limit this for performance because fetching broker
        offsets from Kafka is a relatively inexpensive operation.

        Sends one OffsetRequest per broker to get offsets for all partitions
        where that broker is the leader:
        https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-OffsetAPI(AKAListOffset)

        Can we cleanup connections on agent restart?
        Brokers before 0.9 - accumulate stale connections on restarts.
        In 0.9 Kafka added connections.max.idle.ms
        https://issues.apache.org/jira/browse/KAFKA-1282
        """

        # Connect to Kafka
        highwater_offsets = {}
        topic_partitions_without_a_leader = []
        topics_to_fetch = defaultdict(set)
        cli = self._get_kafka_client(instance)

        for topic, partitions in iteritems(topics):
            # if no partitions are provided
            # we're falling back to all available partitions (?)
            if len(partitions) == 0:
                partitions = cli.cluster.available_partitions_for_topic(topic)
            topics_to_fetch[topic].update(partitions)

        leader_tp = defaultdict(lambda: defaultdict(set))
        for topic, partitions in iteritems(topics_to_fetch):
            for partition in partitions:
                partition_leader = cli.cluster.leader_for_partition(TopicPartition(topic, partition))
                if partition_leader is not None and partition_leader >= 0:
                    leader_tp[partition_leader][topic].add(partition)

        max_offsets = 1
        for node_id, tps in iteritems(leader_tp):
            # Construct the OffsetRequest
            request = OffsetRequest[0](
                replica_id=-1,
                topics=[
                    (topic, [(partition, OffsetResetStrategy.LATEST, max_offsets) for partition in partitions])
                    for topic, partitions in iteritems(tps)
                ],
            )

            response = self._make_blocking_req(cli, request, node_id=node_id)
            offsets, unled = self._process_highwater_offsets(response)
            highwater_offsets.update(offsets)
            topic_partitions_without_a_leader.extend(unled)

        return highwater_offsets, list(set(topic_partitions_without_a_leader))


Instruction:
Here you have a function in Python 3, explain what it does
def _get_zk_path_children(self, zk_conn, zk_path, name_for_error):
        children = []
        try:
            children = zk_conn.get_children(zk_path)
        except NoNodeError:
            self.log.info('No zookeeper node at %s', zk_path)
        except Exception:
            self.log.exception('Could not read %s from %s', name_for_error, zk_path)
        return children

Response: Fetch child nodes for a given Zookeeper path.


Instruction:
Implement a function in Python 3 to
fetch consumer group offsets from Zookeeper.

Response:
def _get_zk_consumer_offsets(self, zk_hosts_ports, consumer_groups=None, zk_prefix=''):
        """
        Fetch Consumer Group offsets from Zookeeper.

        Also fetch consumer_groups, topics, and partitions if not
        already specified in consumer_groups.

        :param dict consumer_groups: The consumer groups, topics, and partitions
            that you want to fetch offsets for. If consumer_groups is None, will
            fetch offsets for all consumer_groups. For examples of what this
            dict can look like, see _validate_explicit_consumer_groups().
        """
        zk_consumer_offsets = {}

        # Construct the Zookeeper path pattern
        # /consumers/[groupId]/offsets/[topic]/[partitionId]
        zk_path_consumer = zk_prefix + '/consumers/'
        zk_path_topic_tmpl = zk_path_consumer + '{group}/offsets/'
        zk_path_partition_tmpl = zk_path_topic_tmpl + '{topic}/'

        zk_conn = KazooClient(zk_hosts_ports, timeout=self._zk_timeout)
        zk_conn.start()
        try:
            if consumer_groups is None:
                # If consumer groups aren't specified, fetch them from ZK
                consumer_groups = {
                    consumer_group: None
                    for consumer_group in self._get_zk_path_children(zk_conn, zk_path_consumer, 'consumer groups')
                }

            for consumer_group, topics in iteritems(consumer_groups):
                if topics is None:
                    # If topics are't specified, fetch them from ZK
                    zk_path_topics = zk_path_topic_tmpl.format(group=consumer_group)
                    topics = {topic: None for topic in self._get_zk_path_children(zk_conn, zk_path_topics, 'topics')}
                    consumer_groups[consumer_group] = topics

                for topic, partitions in iteritems(topics):
                    if partitions is not None:
                        partitions = set(partitions)  # defend against bad user input
                    else:
                        # If partitions aren't specified, fetch them from ZK
                        zk_path_partitions = zk_path_partition_tmpl.format(group=consumer_group, topic=topic)
                        # Zookeeper returns the partition IDs as strings because
                        # they are extracted from the node path
                        partitions = [
                            int(x) for x in self._get_zk_path_children(zk_conn, zk_path_partitions, 'partitions')
                        ]
                        consumer_groups[consumer_group][topic] = partitions

                    # Fetch consumer offsets for each partition from ZK
                    for partition in partitions:
                        zk_path = (zk_path_partition_tmpl + '{partition}/').format(
                            group=consumer_group, topic=topic, partition=partition
                        )
                        try:
                            consumer_offset = int(zk_conn.get(zk_path)[0])
                            key = (consumer_group, topic, partition)
                            zk_consumer_offsets[key] = consumer_offset
                        except NoNodeError:
                            self.log.info('No zookeeper node at %s', zk_path)
                        except Exception:
                            self.log.exception('Could not read consumer offset from %s', zk_path)
        finally:
            try:
                zk_conn.stop()
                zk_conn.close()
            except Exception:
                self.log.exception('Error cleaning up Zookeeper connection')
        return zk_consumer_offsets, consumer_groups


Instruction:
Can you tell what is the following Python 3 function doing
def _get_kafka_consumer_offsets(self, instance, consumer_groups):
        consumer_offsets = {}
        topics = defaultdict(set)

        cli = self._get_kafka_client(instance)

        for consumer_group, topic_partitions in iteritems(consumer_groups):
            try:
                coordinator_id = self._get_group_coordinator(cli, consumer_group)
                if coordinator_id:
                    offsets = self._get_consumer_offsets(cli, consumer_group, topic_partitions, coordinator_id)
                else:
                    offsets = self._get_consumer_offsets(cli, consumer_group, topic_partitions)
                    self.log.info("unable to find group coordinator for %s", consumer_group)

                for (topic, partition), offset in iteritems(offsets):
                    topics[topic].update([partition])
                    key = (consumer_group, topic, partition)
                    consumer_offsets[key] = offset
            except Exception:
                self.log.exception('Could not read consumer offsets from kafka.')

        return consumer_offsets, topics

Response: retrieve consumer offsets via the new consumer api


Instruction:
Can you generate the documentation for the following Python 3 function
def _validate_explicit_consumer_groups(cls, val):
        assert isinstance(val, dict)
        for consumer_group, topics in iteritems(val):
            assert isinstance(consumer_group, string_types)
            # topics are optional
            assert isinstance(topics, dict) or topics is None
            if topics is not None:
                for topic, partitions in iteritems(topics):
                    assert isinstance(topic, string_types)
                    # partitions are optional
                    assert isinstance(partitions, (list, tuple)) or partitions is None
                    if partitions is not None:
                        for partition in partitions:
                            assert isinstance(partition, int)

Response: Validate any explicitly specified consumer groups.


Instruction:
How would you implement a function in Python 3 that
emits an event to the Datadog Event Stream.

Response:
def _send_event(self, title, text, tags, event_type, aggregation_key, severity='info'):
        """Emit an event to the Datadog Event Stream."""
        event_dict = {
            'timestamp': int(time()),
            'source_type_name': self.SOURCE_TYPE_NAME,
            'msg_title': title,
            'event_type': event_type,
            'alert_type': severity,
            'msg_text': text,
            'tags': tags,
            'aggregation_key': aggregation_key,
        }
        self.event(event_dict)


Instruction:
Can you tell what is the following Python 3 function doing
def display_package_changes(pre_packages, post_packages, indent=''):
    # use package name to determine what's changed
    pre_package_names = {p.name: p for p in pre_packages}
    post_package_names = {p.name: p for p in post_packages}

    added = set(post_package_names.keys()) - set(pre_package_names.keys())
    removed = set(pre_package_names.keys()) - set(post_package_names.keys())
    changed_maybe = set(pre_package_names.keys()) & set(post_package_names.keys())

    changed = []
    for package_name in sorted(changed_maybe):
        if pre_package_names[package_name] != post_package_names[package_name]:
            changed.append((pre_package_names[package_name], post_package_names[package_name]))

    if not (added or removed or changed):
        echo_info('{}No changes'.format(indent))

    if added:
        echo_success('{}Added packages:'.format(indent))
        for package_name in sorted(added):
            echo_info('{}    {}'.format(indent, post_package_names[package_name]))

    if removed:
        echo_failure('{}Removed packages:'.format(indent))
        for package_name in sorted(removed):
            echo_info('{}    {}'.format(indent, pre_package_names[package_name]))

    if changed:
        echo_warning('{}Changed packages:'.format(indent))
        for pre, post in changed:
            echo_info('{}    {} -> {}'.format(indent, pre, post))

Response: Print packages that have been added removed or changed.


Instruction:
Create a Python 3 function for
resolving transient dependencies for any number of checks.

Response:
def resolve(checks, lazy, quiet):
    """Resolve transient dependencies for any number of checks.
    If you want to do this en masse, put `all`.
    """
    root = get_root()
    if 'all' in checks:
        checks = os.listdir(root)

    for check_name in sorted(checks):
        pinned_reqs_file = os.path.join(root, check_name, 'requirements.in')
        resolved_reqs_file = os.path.join(root, check_name, 'requirements.txt')

        if os.path.isfile(pinned_reqs_file):
            if not quiet:
                echo_info('Check `{}`:'.format(check_name))

            if not quiet:
                echo_waiting('    Resolving dependencies...')

            pre_packages = read_packages(resolved_reqs_file)
            result = resolve_requirements(pinned_reqs_file, resolved_reqs_file, lazy=lazy)
            if result.code:
                abort(result.stdout + result.stderr)

            if not quiet:
                post_packages = read_packages(resolved_reqs_file)
                display_package_changes(pre_packages, post_packages, indent='    ')


Instruction:
Here you have a function in Python 3, explain what it does
def pin(package, version, checks, marker, resolving, lazy, quiet):
    root = get_root()
    package_name = package.lower()
    version = version.lower()

    for check_name in sorted(os.listdir(root)):
        pinned_reqs_file = os.path.join(root, check_name, 'requirements.in')
        resolved_reqs_file = os.path.join(root, check_name, 'requirements.txt')

        if os.path.isfile(pinned_reqs_file):
            pinned_packages = {package.name: package for package in read_packages(pinned_reqs_file)}
            if package not in pinned_packages and check_name not in checks:
                continue

            if resolving:
                pre_packages = list(read_packages(resolved_reqs_file))
            else:
                pre_packages = list(itervalues(pinned_packages))

            if not quiet:
                echo_info('Check `{}`:'.format(check_name))

            if version == 'none':
                del pinned_packages[package_name]
            else:
                pinned_packages[package_name] = Package(package_name, version, marker)

            package_list = sorted(itervalues(pinned_packages))
            write_file_lines(pinned_reqs_file, ('{}\n'.format(package) for package in package_list))

            if not quiet:
                echo_waiting('    Resolving dependencies...')

            if resolving:
                result = resolve_requirements(pinned_reqs_file, resolved_reqs_file, lazy=lazy)
                if result.code:
                    abort(result.stdout + result.stderr)

            if not quiet:
                post_packages = read_packages(resolved_reqs_file if resolving else pinned_reqs_file)
                display_package_changes(pre_packages, post_packages, indent='    ')

Response: Pin a dependency for all checks that require it.


Instruction:
Write a Python 3 script to
combine all dependencies for the Agent s static environment.

Response:
def freeze():
    """Combine all dependencies for the Agent's static environment."""
    echo_waiting('Verifying collected packages...')
    catalog, errors = make_catalog()
    if errors:
        for error in errors:
            echo_failure(error)
        abort()

    static_file = get_agent_requirements()

    echo_info('Static file: {}'.format(static_file))

    pre_packages = list(read_packages(static_file))

    catalog.write_packages(static_file)

    post_packages = list(read_packages(static_file))
    display_package_changes(pre_packages, post_packages)


Instruction:
Can you write a function in Python 3 where it
restarts an Agent to detect environment changes.

Response:
def reload_env(check, env):
    """Restart an Agent to detect environment changes."""
    envs = get_configured_envs(check)
    if not envs:
        echo_failure('No active environments found for `{}`.'.format(check))
        echo_info('See what is available to start via `ddev env ls {}`.'.format(check))
        abort()

    if not env:
        if len(envs) > 1:
            echo_failure('Multiple active environments found for `{}`, please specify one.'.format(check))
            echo_info('See what is active via `ddev env ls`.')
            abort()

        env = envs[0]

    if env not in envs:
        echo_failure('`{}` is not an active environment.'.format(env))
        echo_info('See what is active via `ddev env ls`.')
        abort()

    environment = create_interface(check, env)

    result = environment.restart_agent()

    if result.code:
        abort(result.stdout + result.stderr, code=result.code)
    else:
        echo_success('Successfully reloaded environment `{}`!'.format(env))


Instruction:
Here you have a function in Python 3, explain what it does
def _create_gitlab_prometheus_instance(self, instance, init_config):
        # Mapping from Prometheus metrics names to Datadog ones
        # For now it's a 1:1 mapping
        allowed_metrics = init_config.get('allowed_metrics')
        if allowed_metrics is None:
            raise CheckException("At least one metric must be whitelisted in `allowed_metrics`.")

        gitlab_instance = deepcopy(instance)
        # gitlab uses 'prometheus_endpoint' and not 'prometheus_url', so we have to rename the key
        gitlab_instance['prometheus_url'] = instance.get('prometheus_endpoint')

        gitlab_instance.update(
            {
                'namespace': 'gitlab',
                'metrics': allowed_metrics,
                # Defaults that were set when gitlab was based on PrometheusCheck
                'send_monotonic_counter': instance.get('send_monotonic_counter', False),
                'health_service_check': instance.get('health_service_check', False),
            }
        )

        return gitlab_instance

Response: Create a gitlab instance based on the instance configuration.


Instruction:
Write a Python 3 function that can
submit a metric as a gauge

Response:
def _submit_gauge(self, metric_name, val, metric, custom_tags=None, hostname=None):
        """
        Submit a metric as a gauge, additional tags provided will be added to
        the ones from the label provided via the metrics object.

        `custom_tags` is an array of 'tag:value' that will be added to the
        metric when sending the gauge to Datadog.
        """
        _tags = self._metric_tags(metric_name, val, metric, custom_tags, hostname)
        self.check.gauge('{}.{}'.format(self.NAMESPACE, metric_name), val, _tags, hostname=hostname)


Instruction:
How would you code a function in Python 3 to
validate the tag sets for a MongoReplicaSetClient.

Response:
def _validate_tag_sets(tag_sets):
    """Validate tag sets for a MongoReplicaSetClient.
    """
    if tag_sets is None:
        return tag_sets

    if not isinstance(tag_sets, list):
        raise TypeError((
            "Tag sets %r invalid, must be a list") % (tag_sets,))
    if len(tag_sets) == 0:
        raise ValueError((
            "Tag sets %r invalid, must be None or contain at least one set of"
            " tags") % (tag_sets,))

    for tags in tag_sets:
        if not isinstance(tags, Mapping):
            raise TypeError(
                "Tag set %r invalid, must be an instance of dict, "
                "bson.son.SON or other type that inherits from "
                "collection.Mapping" % (tags,))

    return tag_sets


Instruction:
Write a Python 3 script for
reading preference as a document.

Response:
def document(self):
        """Read preference as a document.
        """
        doc = {'mode': self.__mongos_mode}
        if self.__tag_sets not in (None, [{}]):
            doc['tags'] = self.__tag_sets
        if self.__max_staleness != -1:
            doc['maxStalenessSeconds'] = self.__max_staleness
        return doc


Instruction:
How would you implement a function in Python 3 that
validates default configuration files.

Response:
def config(check):
    """Validate default configuration files."""
    if check:
        checks = [check]
    else:
        checks = sorted(get_valid_checks())

    files_failed = {}
    files_warned = {}
    num_files = 0

    echo_waiting('Validating default configuration files...')
    for check in checks:
        check_display_queue = []

        config_files = get_config_files(check)
        for config_file in config_files:
            num_files += 1
            file_display_queue = []
            file_name = basepath(config_file)

            try:
                config_data = yaml.safe_load(read_file(config_file))
            except Exception as e:
                files_failed[config_file] = True

                # We must convert to text here to free Exception object before it goes out of scope
                error = str(e)

                check_display_queue.append(lambda: echo_info('{}:'.format(file_name), indent=True))
                check_display_queue.append(lambda: echo_failure('Invalid YAML -', indent=FILE_INDENT))
                check_display_queue.append(lambda: echo_info(error, indent=FILE_INDENT * 2))
                continue

            # Verify there is an `instances` section
            if 'instances' not in config_data:
                files_failed[config_file] = True
                file_display_queue.append(lambda: echo_failure('Missing `instances` section', indent=FILE_INDENT))

            # Verify there is a default instance
            else:
                instances = config_data['instances']
                if check not in IGNORE_DEFAULT_INSTANCE and not isinstance(instances, list):
                    files_failed[config_file] = True
                    file_display_queue.append(lambda: echo_failure('No default instance', indent=FILE_INDENT))

            if file_display_queue:
                check_display_queue.append(lambda: echo_info('{}:'.format(file_name), indent=True))
                check_display_queue.extend(file_display_queue)

        if check_display_queue:
            echo_success('{}:'.format(check))
            for display in check_display_queue:
                display()

    files_failed = len(files_failed)
    files_warned = len(files_warned)
    files_passed = num_files - (files_failed + files_warned)

    if files_failed or files_warned:
        click.echo()

    if files_failed:
        echo_failure('Files with errors: {}'.format(files_failed))

    if files_warned:
        echo_warning('Files with warnings: {}'.format(files_warned))

    if files_passed:
        if files_failed or files_warned:
            echo_success('Files valid: {}'.format(files_passed))
        else:
            echo_success('All {} configuration files are valid!'.format(num_files))

    if files_failed:
        abort()


Instruction:
How would you implement a function in Python 3 that
gets stats from the queue manager

Response:
def queue_manager_stats(self, queue_manager, tags):
        """
        Get stats from the queue manager
        """
        for mname, pymqi_value in iteritems(metrics.queue_manager_metrics()):
            try:
                m = queue_manager.inquire(pymqi_value)

                mname = '{}.queue_manager.{}'.format(self.METRIC_PREFIX, mname)
                self.gauge(mname, m, tags=tags)
                self.service_check(self.QUEUE_MANAGER_SERVICE_CHECK, AgentCheck.OK, tags)
            except pymqi.Error as e:
                self.warning("Error getting queue manager stats: {}".format(e))
                self.service_check(self.QUEUE_MANAGER_SERVICE_CHECK, AgentCheck.CRITICAL, tags)


Instruction:
Implement a Python 3 function for
grabbing stats from queues 69

Response:
def queue_stats(self, queue, tags):
        """
        Grab stats from queues
        """
        for mname, pymqi_value in iteritems(metrics.queue_metrics()):
            try:
                mname = '{}.queue.{}'.format(self.METRIC_PREFIX, mname)
                m = queue.inquire(pymqi_value)
                self.gauge(mname, m, tags=tags)
            except pymqi.Error as e:
                self.warning("Error getting queue stats for {}: {}".format(queue, e))

        for mname, func in iteritems(metrics.queue_metrics_functions()):
            try:
                mname = '{}.queue.{}'.format(self.METRIC_PREFIX, mname)
                m = func(queue)
                self.gauge(mname, m, tags=tags)
            except pymqi.Error as e:
                self.warning("Error getting queue stats for {}: {}".format(queue, e))


Instruction:
Can you generate the documentation for the following Python 3 function
def get_version_string(check_name):
    version = VERSION.search(read_version_file(check_name))
    if version:
        return version.group(1)

Response: Get the version string for the given check.


Instruction:
Write a Python 3 script for
loading the manifest file into a dictionary

Response:
def load_manifest(check_name):
    """
    Load the manifest file into a dictionary
    """
    manifest_path = os.path.join(get_root(), check_name, 'manifest.json')
    if file_exists(manifest_path):
        return json.loads(read_file(manifest_path))
    return {}


Instruction:
Write a Python 3 script to
parse the requirements file contents into a dictionary mapping check - package - name - > pinned_version.

Response:
def parse_agent_req_file(contents):
    """
    Returns a dictionary mapping {check-package-name --> pinned_version} from the
    given file contents. We can assume lines are in the form:

        datadog-active-directory==1.1.1; sys_platform == 'win32'

    """
    catalog = OrderedDict()
    for line in contents.splitlines():
        toks = line.split('==', 1)
        if len(toks) != 2 or not toks[0] or not toks[1]:
            # if we get here, the requirements file is garbled but let's stay
            # resilient
            continue

        name, other = toks
        version = other.split(';')
        catalog[name] = version[0]

    return catalog


Instruction:
Write a Python 3 function for
collecting the dictionary of the current state of the object.

Response:
def _collect_dict(self, metric_type, field_metric_map, query, db, tags):
        """
        Query status and get a dictionary back.
        Extract each field out of the dictionary
        and stuff it in the corresponding metric.

        query: show status...
        field_metric_map: {"Seconds_behind_master": "mysqlSecondsBehindMaster"}
        """
        try:
            with closing(db.cursor()) as cursor:
                cursor.execute(query)
                result = cursor.fetchone()
                if result is not None:
                    for field, metric in list(iteritems(field_metric_map)):
                        # Find the column name in the cursor description to identify the column index
                        # http://www.python.org/dev/peps/pep-0249/
                        # cursor.description is a tuple of (column_name, ..., ...)
                        try:
                            col_idx = [d[0].lower() for d in cursor.description].index(field.lower())
                            self.log.debug("Collecting metric: %s" % metric)
                            if result[col_idx] is not None:
                                self.log.debug("Collecting done, value %s" % result[col_idx])
                                if metric_type == GAUGE:
                                    self.gauge(metric, float(result[col_idx]), tags=tags)
                                elif metric_type == RATE:
                                    self.rate(metric, float(result[col_idx]), tags=tags)
                                else:
                                    self.gauge(metric, float(result[col_idx]), tags=tags)
                            else:
                                self.log.debug("Received value is None for index %d" % col_idx)
                        except ValueError:
                            self.log.exception("Cannot find %s in the columns %s" % (field, cursor.description))
        except Exception:
            self.warning("Error while running %s\n%s" % (query, traceback.format_exc()))
            self.log.exception("Error while running %s" % query)


Instruction:
Can you create a Python 3 function that
gets the pid_file variable of MySQL.

Response:
def _get_pid_file_variable(self, db):
        """
        Get the `pid_file` variable
        """
        pid_file = None
        try:
            with closing(db.cursor()) as cursor:
                cursor.execute("SHOW VARIABLES LIKE 'pid_file'")
                pid_file = cursor.fetchone()[1]
        except Exception:
            self.warning("Error while fetching pid_file variable of MySQL.")

        return pid_file


Instruction:
Can you generate the documentation for the following Python 3 function
def _get_slave_status(self, db, above_560, nonblocking):
        try:
            with closing(db.cursor()) as cursor:
                if above_560 and nonblocking:
                    # Query `performance_schema.threads` instead of `
                    # information_schema.processlist` to avoid mutex impact on performance.
                    cursor.execute("SELECT THREAD_ID, NAME FROM performance_schema.threads WHERE NAME LIKE '%worker'")
                else:
                    cursor.execute("SELECT * FROM INFORMATION_SCHEMA.PROCESSLIST WHERE COMMAND LIKE '%Binlog dump%'")
                slave_results = cursor.fetchall()
                slaves = 0
                for _ in slave_results:
                    slaves += 1

                return {'Slaves_connected': slaves}

        except (pymysql.err.InternalError, pymysql.err.OperationalError) as e:
            self.warning("Privileges error accessing the process tables (must grant PROCESS): %s" % str(e))
            return {}

Response: Retrieve the slaves status using the process tables.


Instruction:
Can you write a function in Python 3 where it
parses the general counters from the init_config and return a dictionary for each job counter

Response:
def _parse_general_counters(self, init_config):
        """
        Return a dictionary for each job counter
        {
          counter_group_name: [
              counter_name
            ]
          }
        }
        """
        job_counter = {}

        if init_config.get('general_counters'):

            # Parse the custom metrics
            for counter_group in init_config['general_counters']:
                counter_group_name = counter_group.get('counter_group_name')
                counters = counter_group.get('counters')

                if not counter_group_name:
                    raise Exception('"general_counters" must contain a valid "counter_group_name"')

                if not counters:
                    raise Exception('"general_counters" must contain a list of "counters"')

                # Add the counter_group to the job_counters if it doesn't already exist
                if counter_group_name not in job_counter:
                    job_counter[counter_group_name] = []

                for counter in counters:
                    counter_name = counter.get('counter_name')

                    if not counter_name:
                        raise Exception('At least one "counter_name" should be specified in the list of "counters"')

                    job_counter[counter_group_name].append(counter_name)

        return job_counter


Instruction:
How would you code a function in Python 3 to
parse the job specific counters.

Response:
def _parse_job_specific_counters(self, init_config):
        """
        Return a dictionary for each job counter
        {
          job_name: {
            counter_group_name: [
                counter_name
              ]
            }
          }
        }
        """
        job_counter = {}

        if init_config.get('job_specific_counters'):

            # Parse the custom metrics
            for job in init_config['job_specific_counters']:
                job_name = job.get('job_name')
                metrics = job.get('metrics')

                if not job_name:
                    raise Exception('Counter metrics must have a "job_name"')

                if not metrics:
                    raise Exception("Jobs specified in counter metrics must contain at least one metric")

                # Add the job to the custom metrics if it doesn't already exist
                if job_name not in job_counter:
                    job_counter[job_name] = {}

                for metric in metrics:
                    counter_group_name = metric.get('counter_group_name')
                    counters = metric.get('counters')

                    if not counter_group_name:
                        raise Exception('Each counter metric must contain a valid "counter_group_name"')

                    if not counters:
                        raise Exception('Each counter metric must contain a list of "counters"')

                    # Add the counter group name if it doesn't exist for the current job
                    if counter_group_name not in job_counter[job_name]:
                        job_counter[job_name][counter_group_name] = []

                    for counter in counters:
                        counter_name = counter.get('counter_name')

                        if not counter_name:
                            raise Exception('At least one "counter_name" should be specified in the list of "counters"')

                        job_counter[job_name][counter_group_name].append(counter_name)

        return job_counter


Instruction:
Here you have a function in Python 3, explain what it does
def _get_running_app_ids(self, rm_address, auth, ssl_verify):
        metrics_json = self._rest_request_to_json(
            rm_address,
            auth,
            ssl_verify,
            self.YARN_APPS_PATH,
            self.YARN_SERVICE_CHECK,
            states=self.YARN_APPLICATION_STATES,
            applicationTypes=self.YARN_APPLICATION_TYPES,
        )

        running_apps = {}

        if metrics_json.get('apps'):
            if metrics_json['apps'].get('app') is not None:

                for app_json in metrics_json['apps']['app']:
                    app_id = app_json.get('id')
                    tracking_url = app_json.get('trackingUrl')
                    app_name = app_json.get('name')

                    if app_id and tracking_url and app_name:
                        running_apps[app_id] = (app_name, tracking_url)

        return running_apps

Response: Get the running MapReduce applications.


Instruction:
How would you explain what the following Python 3 function does
def _mapreduce_job_metrics(self, running_apps, auth, ssl_verify, addl_tags):
        running_jobs = {}

        for app_name, tracking_url in itervalues(running_apps):

            metrics_json = self._rest_request_to_json(
                tracking_url, auth, ssl_verify, self.MAPREDUCE_JOBS_PATH, self.MAPREDUCE_SERVICE_CHECK
            )

            if metrics_json.get('jobs'):
                if metrics_json['jobs'].get('job'):

                    for job_json in metrics_json['jobs']['job']:
                        job_id = job_json.get('id')
                        job_name = job_json.get('name')
                        user_name = job_json.get('user')

                        if job_id and job_name and user_name:

                            # Build the structure to hold the information for each job ID
                            running_jobs[str(job_id)] = {
                                'job_name': str(job_name),
                                'app_name': str(app_name),
                                'user_name': str(user_name),
                                'tracking_url': self._join_url_dir(tracking_url, self.MAPREDUCE_JOBS_PATH, job_id),
                            }

                            tags = [
                                'app_name:' + str(app_name),
                                'user_name:' + str(user_name),
                                'job_name:' + str(job_name),
                            ]

                            tags.extend(addl_tags)

                            self._set_metrics_from_json(job_json, self.MAPREDUCE_JOB_METRICS, tags)

        return running_jobs

Response: Get metrics for each MapReduce job.


Instruction:
How would you explain what the following Python 3 function does
def _mapreduce_job_counters_metrics(self, running_jobs, auth, ssl_verify, addl_tags):
        for job_metrics in itervalues(running_jobs):
            job_name = job_metrics['job_name']

            # Check if the job_name exist in the custom metrics
            if self.general_counters or (job_name in self.job_specific_counters):
                job_specific_metrics = self.job_specific_counters.get(job_name)

                metrics_json = self._rest_request_to_json(
                    job_metrics['tracking_url'],
                    auth,
                    ssl_verify,
                    'counters',
                    self.MAPREDUCE_SERVICE_CHECK,
                    tags=addl_tags,
                )

                if metrics_json.get('jobCounters'):
                    if metrics_json['jobCounters'].get('counterGroup'):

                        # Cycle through all the counter groups for this job
                        for counter_group in metrics_json['jobCounters']['counterGroup']:
                            group_name = counter_group.get('counterGroupName')

                            if group_name:
                                counter_metrics = set([])

                                # Add any counters in the job specific metrics
                                if job_specific_metrics and group_name in job_specific_metrics:
                                    counter_metrics = counter_metrics.union(job_specific_metrics[group_name])

                                # Add any counters in the general metrics
                                if group_name in self.general_counters:
                                    counter_metrics = counter_metrics.union(self.general_counters[group_name])

                                if counter_metrics:
                                    # Cycle through all the counters in this counter group
                                    if counter_group.get('counter'):
                                        for counter in counter_group['counter']:
                                            counter_name = counter.get('name')

                                            # Check if the counter name is in the custom metrics for this group name
                                            if counter_name and counter_name in counter_metrics:
                                                tags = [
                                                    'app_name:' + job_metrics.get('app_name'),
                                                    'user_name:' + job_metrics.get('user_name'),
                                                    'job_name:' + job_name,
                                                    'counter_name:' + str(counter_name).lower(),
                                                ]

                                                tags.extend(addl_tags)

                                                self._set_metrics_from_json(
                                                    counter, self.MAPREDUCE_JOB_COUNTER_METRICS, tags
                                                )

Response: Get the custom metrics specified for each counter group and add them to the running jobs list


Instruction:
Write a Python 3 script for
getting metrics for each MapReduce task

Response:
def _mapreduce_task_metrics(self, running_jobs, auth, ssl_verify, addl_tags):
        """
        Get metrics for each MapReduce task
        Return a dictionary of {task_id: 'tracking_url'} for each MapReduce task
        """
        for job_stats in itervalues(running_jobs):

            metrics_json = self._rest_request_to_json(
                job_stats['tracking_url'], auth, ssl_verify, 'tasks', self.MAPREDUCE_SERVICE_CHECK, tags=addl_tags
            )

            if metrics_json.get('tasks'):
                if metrics_json['tasks'].get('task'):

                    for task in metrics_json['tasks']['task']:
                        task_type = task.get('type')

                        if task_type:
                            tags = [
                                'app_name:' + job_stats['app_name'],
                                'user_name:' + job_stats['user_name'],
                                'job_name:' + job_stats['job_name'],
                                'task_type:' + str(task_type).lower(),
                            ]

                            tags.extend(addl_tags)

                            if task_type == 'MAP':
                                self._set_metrics_from_json(task, self.MAPREDUCE_MAP_TASK_METRICS, tags)

                            elif task_type == 'REDUCE':
                                self._set_metrics_from_json(task, self.MAPREDUCE_REDUCE_TASK_METRICS, tags)


Instruction:
Write a Python 3 function that can
parse the JSON response and set the metrics

Response:
def _set_metrics_from_json(self, metrics_json, metrics, tags):
        """
        Parse the JSON response and set the metrics
        """
        for status, (metric_name, metric_type) in iteritems(metrics):
            metric_status = metrics_json.get(status)

            if metric_status is not None:
                self._set_metric(metric_name, metric_type, metric_status, tags=tags)


Instruction:
How would you implement a function in Python 3 that
sets a metric in the cache.

Response:
def _set_metric(self, metric_name, metric_type, value, tags=None, device_name=None):
        """
        Set a metric
        """
        if metric_type == self.HISTOGRAM:
            self.histogram(metric_name, value, tags=tags, device_name=device_name)
        elif metric_type == self.INCREMENT:
            self.increment(metric_name, value, tags=tags, device_name=device_name)
        else:
            self.log.error('Metric type "{}" unknown'.format(metric_type))


Instruction:
Here you have a function in Python 3, explain what it does
def _rest_request_to_json(self, address, auth, ssl_verify, object_path, service_name, tags=None, *args, **kwargs):
        response_json = None
        tags = [] if tags is None else tags

        service_check_tags = ['url:{}'.format(self._get_url_base(address))] + tags

        url = address

        if object_path:
            url = self._join_url_dir(url, object_path)

        # Add args to the url
        if args:
            for directory in args:
                url = self._join_url_dir(url, directory)

        self.log.debug('Attempting to connect to "{}"'.format(url))

        # Add kwargs as arguments
        if kwargs:
            query = '&'.join(['{}={}'.format(key, value) for key, value in iteritems(kwargs)])
            url = urljoin(url, '?' + query)

        try:
            response = requests.get(url, auth=auth, verify=ssl_verify, timeout=self.default_integration_http_timeout)
            response.raise_for_status()
            response_json = response.json()

        except Timeout as e:
            self.service_check(
                service_name,
                AgentCheck.CRITICAL,
                tags=service_check_tags,
                message="Request timeout: {}, {}".format(url, e),
            )
            raise

        except (HTTPError, InvalidURL, ConnectionError) as e:
            self.service_check(
                service_name,
                AgentCheck.CRITICAL,
                tags=service_check_tags,
                message="Request failed: {}, {}".format(url, e),
            )
            raise

        except JSONDecodeError as e:
            self.service_check(
                service_name,
                AgentCheck.CRITICAL,
                tags=service_check_tags,
                message="JSON Parse failed: {}, {}".format(url, e),
            )
            raise

        except ValueError as e:
            self.service_check(service_name, AgentCheck.CRITICAL, tags=service_check_tags, message=str(e))
            raise

        return response_json

Response: Perform a REST request to the given URL and return the JSON response.


Instruction:
How would you explain what the following Python 3 function does
def _psutil_config_to_stats(self, instance):
        process_metrics = instance.get('process_metrics', self.init_config.get('process_metrics', None))
        if not process_metrics:
            self.log.error('No metrics configured for AgentMetrics check!')
            return {}

        methods, metric_types = zip(
            *[(p['name'], p.get('type', GAUGE))
                for p in process_metrics if _is_affirmative(p.get('active'))]
        )

        names_to_metric_types = {}
        for i, m in enumerate(methods):
            names_to_metric_types[AgentMetrics._get_statistic_name_from_method(m)] = metric_types[i]

        stats = AgentMetrics._collect_internal_stats(methods)
        return stats, names_to_metric_types

Response: Reads the init_config for the current process_metrics and returns the stats and the internal stats


Instruction:
Can you create a Python 3 function that
registers sample metrics from psutil.

Response:
def _register_psutil_metrics(self, stats, names_to_metric_types, tags=None):
        """
        Saves sample metrics from psutil

        :param stats: a dictionary that looks like:
        {
         'memory_info': OrderedDict([('rss', 24395776), ('vms', 144666624)]),
         'io_counters': OrderedDict([('read_count', 4536),
                                    ('write_count', 100),
                                    ('read_bytes', 0),
                                    ('write_bytes', 61440)])
         ...
         }

         This creates a metric like `datadog.agent.collector.{key_1}.{key_2}` where key_1 is a top-level
         key in `stats`, and key_2 is a nested key.
         E.g. datadog.agent.collector.memory_info.rss
        """
        if tags is None:
            tags = []
        base_metric = 'datadog.agent.collector.{0}.{1}'
        # TODO: May have to call self.normalize(metric_name) to get a compliant name
        for k, v in stats.iteritems():
            metric_type = names_to_metric_types[k]
            if isinstance(v, dict):
                for _k, _v in v.iteritems():
                    full_metric_name = base_metric.format(k, _k)
                    self._send_single_metric(full_metric_name, _v, metric_type, tags)
            else:
                full_metric_name = 'datadog.agent.collector.{0}'.format(k)
                self._send_single_metric(full_metric_name, v, metric_type, tags)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_container_ip(container_id_or_name):
    command = [
        'docker',
        'inspect',
        '-f',
        '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}',
        container_id_or_name,
    ]

    return run_command(command, capture='out', check=True).stdout.strip()

Response: Get a Docker container s IP address from its id or name.


Instruction:
Can you generate the documentation for the following Python 3 function
def docker_run(
    compose_file=None,
    build=False,
    service_name=None,
    up=None,
    down=None,
    sleep=None,
    endpoints=None,
    log_patterns=None,
    conditions=None,
    env_vars=None,
    wrapper=None,
):
    """This utility provides a convenient way to safely set up and tear down Docker environments.

    :param compose_file: A path to a Docker compose file. A custom tear
                         down is not required when using this.
    :type compose_file: ``str``
    :param build: Whether or not to build images for when ``compose_file`` is provided.
    :type build: ``bool``
    :param service_name: Optional name for when ``compose_file`` is provided.
    :type service_name: ``str``
    :param up: A custom setup callable.
    :type up: ``callable``
    :param down: A custom tear down callable. This is required when using a custom setup.
    :type down: ``callable``
    :param sleep: Number of seconds to wait before yielding.
    :type sleep: ``float``
    :param endpoints: Endpoints to verify access for before yielding. Shorthand for adding
                      ``conditions.CheckEndpoints(endpoints)`` to the ``conditions`` argument.
    :type endpoints: ``list`` of ``str``, or a single ``str``
    :param log_patterns: Patterns to find in Docker logs before yielding. This is only available
                         when ``compose_file`` is provided. Shorthand for adding
                         ``conditions.CheckDockerLogs(compose_file, log_patterns)`` to the ``conditions`` argument.
    :type log_patterns: ``list`` of (``str`` or ``re.Pattern``)
    :param conditions: A list of callable objects that will be executed before yielding to check for errors.
    :type conditions: ``callable``
    :param env_vars: A dictionary to update ``os.environ`` with during execution.
    :type env_vars: ``dict``
    :param wrapper: A context manager to use during execution.
    """
    if compose_file and up:
        raise TypeError('You must select either a compose file or a custom setup callable, not both.')

    if compose_file is not None:
        if not isinstance(compose_file, string_types):
            raise TypeError('The path to the compose file is not a string: {}'.format(repr(compose_file)))

        set_up = ComposeFileUp(compose_file, build=build, service_name=service_name)
        if down is not None:
            tear_down = down
        else:
            tear_down = ComposeFileDown(compose_file)
    else:
        set_up = up
        tear_down = down

    docker_conditions = []

    if log_patterns is not None:
        if compose_file is None:
            raise ValueError(
                'The `log_patterns` convenience is unavailable when using '
                'a custom setup. Please use a custom condition instead.'
            )
        docker_conditions.append(CheckDockerLogs(compose_file, log_patterns))

    if conditions is not None:
        docker_conditions.extend(conditions)

    with environment_run(
        up=set_up,
        down=tear_down,
        sleep=sleep,
        endpoints=endpoints,
        conditions=docker_conditions,
        env_vars=env_vars,
        wrapper=wrapper,
    ) as result:
        yield result

Response: This utility is used to set up and tear down Docker environments.


Instruction:
Write a Python 3 function that can
return the SON document representation of this DBRef.

Response:
def as_doc(self):
        """Get the SON document representation of this DBRef.

        Generally not needed by application developers
        """
        doc = SON([("$ref", self.collection),
                   ("$id", self.id)])
        if self.database is not None:
            doc["$db"] = self.database
        doc.update(self.__kwargs)
        return doc


Instruction:
Can you generate the documentation for the following Python 3 function
def clear(self):
        with self._lock:
            self._config = {
                CacheConfig.Morlist: {'last': defaultdict(float), 'intl': {}},
                CacheConfig.Metadata: {'last': defaultdict(float), 'intl': {}},
            }

Response: Reset the config object to its initial state


Instruction:
Create a Python 3 function for
getting the last value for a given key in the configuration.

Response:
def get_last(self, type_, key):
        """
        Notice: this will return the defaultdict default value also for keys
        that are not in the configuration, this is a tradeoff to keep the code simple.
        """
        self._check_type(type_)
        with self._lock:
            return self._config[type_]['last'][key]


Instruction:
Write a Python 3 script for
getting a list of possible paths containing certificates

Response:
def _get_ca_certs_paths():
    """
    Get a list of possible paths containing certificates

    Check is installed via pip to:
     * Windows: embedded/lib/site-packages/datadog_checks/http_check
     * Linux: embedded/lib/python2.7/site-packages/datadog_checks/http_check

    Certificate is installed to:
     * embedded/ssl/certs/cacert.pem

    walk up to `embedded`, and back down to ssl/certs to find the certificate file
    """
    ca_certs = []

    embedded_root = os.path.dirname(os.path.abspath(__file__))
    for _ in range(10):
        if os.path.basename(embedded_root) == 'embedded':
            ca_certs.append(os.path.join(embedded_root, 'ssl', 'certs', 'cacert.pem'))
            break
        embedded_root = os.path.dirname(embedded_root)
    else:
        raise OSError(
            'Unable to locate `embedded` directory. Please specify ca_certs in your http yaml configuration file.'
        )

    try:
        import tornado
    except ImportError:
        # if `tornado` is not present, simply ignore its certificates
        pass
    else:
        ca_certs.append(os.path.join(os.path.dirname(tornado.__file__), 'ca-certificates.crt'))

    ca_certs.append('/etc/ssl/certs/ca-certificates.crt')

    return ca_certs


Instruction:
Can you create a Python 3 function that
validates logo files. Specifying no check will validate all logos.

Response:
def logos(check):

    """Validate logo files. Specifying no check will validate all logos"""

    valid_checks = get_valid_integrations()
    if check:
        if check in valid_checks:
            checks = [check]
        else:
            echo_info('{} is not an integration.'.format(check))
            return
    else:
        checks = sorted(valid_checks)

    blacklisted_integrations_msg = ''
    count_successful = 0
    error_checks = set()

    for check in checks:
        errors = dict()
        display_name = load_manifest(check).get('display_name', check)
        if check in NOT_TILES:
            blacklisted_integrations_msg += '{} does not currently have an integration tile.\n'.format(display_name)
            continue

        path_to_check_logos = os.path.join(get_root(), check, 'logos')

        for logo, required_size in REQUIRED_IMAGES.items():
            logo_file_name = os.path.join(path_to_check_logos, logo)
            if not os.path.isfile(logo_file_name):
                errors[logo] = '    {} is missing for {}'.format(logo, display_name)
            else:
                size = get_resolution(logo_file_name)
                if size != required_size:
                    errors[logo] = '    {} has improper resolution: {}. Should be {}'.format(logo, size, required_size)

        if errors:
            echo_waiting('{}:'.format(display_name))
            echo_failure('\n'.join(errors.values()))
            error_checks.add(check)
        else:
            count_successful += 1

    blacklisted_integrations_msg = blacklisted_integrations_msg.rstrip()

    if error_checks:
        echo_success(blacklisted_integrations_msg)
        abort()
    elif len(checks) == 1:
        if blacklisted_integrations_msg:
            echo_success(blacklisted_integrations_msg)
        else:
            echo_success('Congrats, all {} logos are valid!'.format(display_name))
    else:
        echo_success(
            'Congrats, all {} checks\' logo files are valid! {} checks were blacklisted and skipped.'.format(
                count_successful, len(NOT_TILES)
            )
        )


Instruction:
Given the following Python 3 function, write the documentation
def _parse_varnishstat(self, output, varnishstat_format, tags=None):
        tags = tags or []
        # FIXME: this check is processing an unbounded amount of data
        # we should explicitly list the metrics we want to get from the check
        if varnishstat_format == "xml":
            p = xml.parsers.expat.ParserCreate()
            p.StartElementHandler = self._start_element
            p.EndElementHandler = lambda name: self._end_element(name, tags)
            p.CharacterDataHandler = self._char_data
            self._reset()
            p.Parse(output, True)
        elif varnishstat_format == "json":
            json_output = json.loads(output)
            for name, metric in iteritems(json_output):
                if not isinstance(metric, dict):  # skip 'timestamp' field
                    continue

                if name.startswith("MAIN."):
                    name = name.split('.', 1)[1]
                value = metric.get("value", 0)

                if metric["flag"] in ("a", "c"):
                    self.rate(self.normalize(name, prefix="varnish"), long(value), tags=tags)
                elif metric["flag"] in ("g", "i"):
                    self.gauge(self.normalize(name, prefix="varnish"), long(value), tags=tags)
                    if 'n_purges' in self.normalize(name, prefix="varnish"):
                        self.rate('varnish.n_purgesps', long(value), tags=tags)
        elif varnishstat_format == "text":
            for line in output.split("\n"):
                self.log.debug("Parsing varnish results: %s" % line)
                fields = line.split()
                if len(fields) < 3:
                    break
                name, gauge_val, rate_val = fields[0], fields[1], fields[2]
                metric_name = self.normalize(name, prefix="varnish")

                # Now figure out which value to pick
                if rate_val.lower() in ("nan", "."):
                    # col 2 matters
                    self.log.debug("Varnish (gauge) %s %d" % (metric_name, int(gauge_val)))
                    self.gauge(metric_name, int(gauge_val), tags=tags)
                    if 'n_purges' in metric_name:
                        self.rate('varnish.n_purgesps', float(gauge_val), tags=tags)
                else:
                    # col 3 has a rate (since restart)
                    self.log.debug("Varnish (rate) %s %d" % (metric_name, int(gauge_val)))
                    self.rate(metric_name, float(gauge_val), tags=tags)

Response: Parse the output of the varnishstat command and add it to the internal object.


Instruction:
Write a Python 3 script for
parsing out service checks from varnishadm output.

Response:
def _parse_varnishadm(self, output, tags):
        """ Parse out service checks from varnishadm.

        Example output:

            Backend b0 is Sick
            Current states  good:  2 threshold:  3 window:  5
            Average responsetime of good probes: 0.000000
            Oldest                                                    Newest
            ================================================================
            -------------------------------------------------------------444 Good IPv4
            -------------------------------------------------------------XXX Good Xmit
            -------------------------------------------------------------RRR Good Recv
            ----------------------------------------------------------HHH--- Happy
            Backend b1 is Sick
            Current states  good:  2 threshold:  3 window:  5
            Average responsetime of good probes: 0.000000
            Oldest                                                    Newest
            ================================================================
            ----------------------------------------------------------HHH--- Happy

        Example output (new output format):

            Backend name                   Admin      Probe
            boot.default                   probe      Healthy (no probe)
            boot.backend2                  probe      Healthy 4/4
              Current states  good:  4 threshold:  3 window:  4
              Average response time of good probes: 0.002504
              Oldest ================================================== Newest
              --------------------------------------------------------------44 Good IPv4
              --------------------------------------------------------------XX Good Xmit
              --------------------------------------------------------------RR Good Recv
              ------------------------------------------------------------HHHH Happy

        """
        # Process status by backend.
        backends_by_status = defaultdict(list)
        for line in output.split("\n"):
            backend, status, message = None, None, None
            # split string and remove all empty fields
            tokens = filter(None, line.strip().split(' '))
            tokens = [t for t in tokens]
            if len(tokens):
                if tokens == ['Backend', 'name', 'Admin', 'Probe']:
                    # skip the column headers that exist in new output format
                    continue
                # parse new output format
                # the backend name will include the vcl name
                # so split on first . to remove prefix
                elif len(tokens) >= 4 and tokens[1] in ['healthy', 'sick']:
                    # If the backend health was overriden, lets grab the
                    # overriden value instead of the probed health
                    backend = tokens[0].split('.', 1)[-1]
                    status = tokens[1].lower()
                elif len(tokens) >= 4 and tokens[1] == 'probe':
                    backend = tokens[0].split('.', 1)[-1]
                    status = tokens[2].lower()
                # Parse older Varnish backend output
                elif tokens[0] == 'Backend':
                    backend = tokens[1]
                    status = tokens[-1].lower()

                if tokens[0] == 'Current' and backend is not None:
                    try:
                        message = ' '.join(tokens[2:]).strip()
                    except Exception:
                        # If we can't parse a message still send a status.
                        self.log.exception('Error when parsing message from varnishadm')
                        message = ''

                if backend is not None:
                    backends_by_status[status].append((backend, message))

        for status, backends in iteritems(backends_by_status):
            check_status = BackendStatus.to_check_status(status)
            for backend, message in backends:
                service_checks_tags = ['backend:%s' % backend] + tags
                self.service_check(self.SERVICE_CHECK_NAME, check_status, tags=service_checks_tags, message=message)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def parse_metric_family(self, response):
        if 'application/vnd.google.protobuf' in response.headers['Content-Type']:
            n = 0
            buf = response.content
            while n < len(buf):
                msg_len, new_pos = _DecodeVarint32(buf, n)
                n = new_pos
                msg_buf = buf[n : n + msg_len]
                n += msg_len

                message = metrics_pb2.MetricFamily()
                message.ParseFromString(msg_buf)
                message.name = self.remove_metric_prefix(message.name)

                # Lookup type overrides:
                if self.type_overrides and message.name in self.type_overrides:
                    new_type = self.type_overrides[message.name]
                    if new_type in self.METRIC_TYPES:
                        message.type = self.METRIC_TYPES.index(new_type)
                    else:
                        self.log.debug("type override %s for %s is not a valid type name" % (new_type, message.name))
                yield message

        elif 'text/plain' in response.headers['Content-Type']:
            input_gen = response.iter_lines(chunk_size=self.REQUESTS_CHUNK_SIZE, decode_unicode=True)
            if self._text_filter_blacklist:
                input_gen = self._text_filter_input(input_gen)

            messages = defaultdict(list)  # map with the name of the element (before the labels)
            # and the list of occurrences with labels and values

            obj_map = {}  # map of the types of each metrics
            obj_help = {}  # help for the metrics
            for metric in text_fd_to_metric_families(input_gen):
                metric.name = self.remove_metric_prefix(metric.name)
                metric_name = "%s_bucket" % metric.name if metric.type == "histogram" else metric.name
                metric_type = self.type_overrides.get(metric_name, metric.type)
                if metric_type == "untyped" or metric_type not in self.METRIC_TYPES:
                    continue

                for sample in metric.samples:
                    if (sample[0].endswith("_sum") or sample[0].endswith("_count")) and metric_type in [
                        "histogram",
                        "summary",
                    ]:
                        messages[sample[0]].append({"labels": sample[1], 'value': sample[2]})
                    else:
                        messages[metric_name].append({"labels": sample[1], 'value': sample[2]})

                obj_map[metric.name] = metric_type
                obj_help[metric.name] = metric.documentation

            for _m in obj_map:
                if _m in messages or (obj_map[_m] == 'histogram' and ('{}_bucket'.format(_m) in messages)):
                    yield self._extract_metric_from_map(_m, messages, obj_map, obj_help)
        else:
            raise UnknownFormatError('Unsupported content-type provided: {}'.format(response.headers['Content-Type']))

Response: Parse the MetricFamily from a valid requests. Response object to provide a MetricFamily object.


Instruction:
Can you write a function in Python 3 where it
filters out the text input line by line to avoid parsing and processing metrics.

Response:
def _text_filter_input(self, input_gen):
        """
        Filters out the text input line by line to avoid parsing and processing
        metrics we know we don't want to process. This only works on `text/plain`
        payloads, and is an INTERNAL FEATURE implemented for the kubelet check
        :param input_get: line generator
        :output: generator of filtered lines
        """
        for line in input_gen:
            for item in self._text_filter_blacklist:
                if item in line:
                    break
            else:
                # No blacklist matches, passing the line through
                yield line


Instruction:
Create a Python 3 function for
getting the value of the metric_name matched by the labels in the _metric dictionary.

Response:
def get_metric_value_by_labels(messages, _metric, _m, metric_suffix):
        """
        :param messages: dictionary as metric_name: {labels: {}, value: 10}
        :param _metric: dictionary as {labels: {le: '0.001', 'custom': 'value'}}
        :param _m: str as metric name
        :param metric_suffix: str must be in (count or sum)
        :return: value of the metric_name matched by the labels
        """
        metric_name = '{}_{}'.format(_m, metric_suffix)
        expected_labels = set(
            [(k, v) for k, v in iteritems(_metric["labels"]) if k not in PrometheusScraperMixin.UNWANTED_LABELS]
        )
        for elt in messages[metric_name]:
            current_labels = set(
                [(k, v) for k, v in iteritems(elt["labels"]) if k not in PrometheusScraperMixin.UNWANTED_LABELS]
            )
            # As we have two hashable objects we can compare them without any side effects
            if current_labels == expected_labels:
                return float(elt["value"])

        raise AttributeError("cannot find expected labels for metric %s with suffix %s" % (metric_name, metric_suffix))