Instruction:
How would you explain what the following Python 3 function does
def recordHostname(self, basedir):
        "Record my hostname in twistd.hostname, for user convenience"
        log.msg("recording hostname in twistd.hostname")
        filename = os.path.join(basedir, "twistd.hostname")

        try:
            hostname = os.uname()[1]  # only on unix
        except AttributeError:
            # this tends to fail on non-connected hosts, e.g., laptops
            # on planes
            hostname = socket.getfqdn()

        try:
            with open(filename, "w") as f:
                f.write("{0}\n".format(hostname))
        except Exception:
            log.msg("failed - ignoring")

Response: Record my hostname in twistd. hostname for user convenience


Instruction:
Write a Python 3 function that can
walk upwards from the current directory until we find this topfile

Response:
def getTopdir(topfile, start=None):
    """walk upwards from the current directory until we find this topfile"""
    if not start:
        start = os.getcwd()
    here = start
    toomany = 20
    while toomany > 0:
        if os.path.exists(os.path.join(here, topfile)):
            return here
        next = os.path.dirname(here)
        if next == here:
            break                       # we've hit the root
        here = next
        toomany -= 1
    output("Unable to find topfile '{}' anywhere "
           "from {} upwards".format(topfile, start))
    sys.exit(1)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def dovc(self, cmd):
        env = os.environ.copy()
        env['LC_ALL'] = "C"
        d = utils.getProcessOutputAndValue(self.exe, cmd, env=env,
                                           path=self.treetop)
        d.addCallback(self._didvc, cmd)
        return d

Response:
This accepts the arguments of a command without the actual
        command itself.


Instruction:
How would you code a function in Python 3 to
return a Deferred that fires with a SourceStamp instance.

Response:
def get(self):
        """Return a Deferred that fires with a SourceStamp instance."""
        d = self.getBaseRevision()
        d.addCallback(self.getPatch)
        d.addCallback(self.done)
        return d


Instruction:
Create a Python 3 function for
versioning of check_output which does not throw error

Response:
def check_output(cmd):
    """Version of check_output which does not throw error"""
    popen = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
    out = popen.communicate()[0].strip()
    if not isinstance(out, str):
        out = out.decode(sys.stdout.encoding)
    return out


Instruction:
Can you tell what is the following Python 3 function doing
def getVersionFromArchiveId(git_archive_id='$Format:%ct %d$'):
    # mangle the magic string to make sure it is not replaced by git archive
    if not git_archive_id.startswith('$For''mat:'):
        # source was modified by git archive, try to parse the version from
        # the value of git_archive_id

        match = re.search(r'tag:\s*v([^,)]+)', git_archive_id)
        if match:
            # archived revision is tagged, use the tag
            return gitDescribeToPep440(match.group(1))

        # archived revision is not tagged, use the commit date
        tstamp = git_archive_id.strip().split()[0]
        d = datetime.datetime.utcfromtimestamp(int(tstamp))
        return d.strftime('%Y.%m.%d')
    return None

Response: Extract the version of a source from git archive.


Instruction:
How would you implement a function in Python 3 that
returns the version of the current build.

Response:
def getVersion(init_file):
    """
    Return BUILDBOT_VERSION environment variable, content of VERSION file, git
    tag or 'latest'
    """

    try:
        return os.environ['BUILDBOT_VERSION']
    except KeyError:
        pass

    try:
        cwd = os.path.dirname(os.path.abspath(init_file))
        fn = os.path.join(cwd, 'VERSION')
        with open(fn) as f:
            return f.read().strip()
    except IOError:
        pass

    version = getVersionFromArchiveId()
    if version is not None:
        return version

    try:
        p = Popen(['git', 'describe', '--tags', '--always'], stdout=PIPE, stderr=STDOUT, cwd=cwd)
        out = p.communicate()[0]

        if (not p.returncode) and out:
            v = gitDescribeToPep440(str(out))
            if v:
                return v
    except OSError:
        pass

    try:
        # if we really can't find the version, we use the date of modification of the most recent file
        # docker hub builds cannot use git describe
        return mTimeVersion(init_file)
    except Exception:
        # bummer. lets report something
        return "latest"


Instruction:
Given the following Python 3 function, write the documentation
def _get_commit_msg(self, repo, sha):
        '''
        :param repo: the repo full name, ``{owner}/{project}``.
            e.g. ``buildbot/buildbot``
        '''
        headers = {
            'User-Agent': 'Buildbot'
        }
        if self._token:
            headers['Authorization'] = 'token ' + self._token

        url = '/repos/{}/commits/{}'.format(repo, sha)
        http = yield httpclientservice.HTTPClientService.getService(
            self.master, self.github_api_endpoint, headers=headers,
            debug=self.debug, verify=self.verify)
        res = yield http.get(url)
        data = yield res.json()
        msg = data.get('commit', {'message': 'No message field'})['message']
        return msg

Response: Get the message of a commit.


Instruction:
Can you generate the documentation for the following Python 3 function
def _process_change(self, payload, user, repo, repo_url, project, event,
                        properties):
        """
        Consumes the JSON as a python object and actually starts the build.

        :arguments:
            payload
                Python Object that represents the JSON sent by GitHub Service
                Hook.
        """
        changes = []
        refname = payload['ref']

        # We only care about regular heads or tags
        match = re.match(r"^refs/(heads|tags)/(.+)$", refname)
        if not match:
            log.msg("Ignoring refname `{}': Not a branch".format(refname))
            return changes
        category = None  # None is the legacy category for when hook only supported push
        if match.group(1) == "tags":
            category = "tag"

        branch = match.group(2)
        if payload.get('deleted'):
            log.msg("Branch `{}' deleted, ignoring".format(branch))
            return changes

        # check skip pattern in commit message. e.g.: [ci skip] and [skip ci]
        head_msg = payload['head_commit'].get('message', '')
        if self._has_skip(head_msg):
            return changes
        commits = payload['commits']
        if payload.get('created'):
            commits = [payload['head_commit']]
        for commit in commits:
            files = []
            for kind in ('added', 'modified', 'removed'):
                files.extend(commit.get(kind, []))

            when_timestamp = dateparse(commit['timestamp'])

            log.msg("New revision: {}".format(commit['id'][:8]))

            change = {
                'author': '{} <{}>'.format(commit['author']['name'],
                                           commit['author']['email']),
                'files': files,
                'comments': commit['message'],
                'revision': commit['id'],
                'when_timestamp': when_timestamp,
                'branch': branch,
                'revlink': commit['url'],
                'repository': repo_url,
                'project': project,
                'properties': {
                    'github_distinct': commit.get('distinct', True),
                    'event': event,
                },
                'category': category
            }
            # Update with any white-listed github event properties
            change['properties'].update(properties)

            if callable(self._codebase):
                change['codebase'] = self._codebase(payload)
            elif self._codebase is not None:
                change['codebase'] = self._codebase

            changes.append(change)

        return changes

Response: Processes a change from GitHub Service and returns a list of dictionaries that represents the changes.


Instruction:
Write a Python 3 function for
checking if the message contains the skipping keyword.

Response:
def _has_skip(self, msg):
        '''
        The message contains the skipping keyword no not.

        :return type: Bool
        '''
        for skip in self.skips:
            if re.search(skip, msg):
                return True
        return False


Instruction:
Given the following Python 3 function, write the documentation
def createSummary(self, log):
        warnings = self.obs.warnings
        errors = self.obs.errors

        if warnings:
            self.addCompleteLog('%d Warnings' % len(warnings), "\n".join(warnings))
            self.warnCount = len(warnings)
        if errors:
            self.addCompleteLog('%d Errors' % len(errors), "\n".join(errors))
            self.errCount = len(errors)

Response: Create nice summary logs.


Instruction:
Can you write a function in Python 3 where it
writes a block of data to the remote writer

Response:
def _writeBlock(self):
        """Write a block of data to the remote writer"""

        if self.interrupted or self.fp is None:
            if self.debug:
                log.msg('WorkerFileUploadCommand._writeBlock(): end')
            return True

        length = self.blocksize
        if self.remaining is not None and length > self.remaining:
            length = self.remaining

        if length <= 0:
            if self.stderr is None:
                self.stderr = 'Maximum filesize reached, truncating file \'{0}\''.format(
                    self.path)
                self.rc = 1
            data = ''
        else:
            data = self.fp.read(length)

        if self.debug:
            log.msg('WorkerFileUploadCommand._writeBlock(): ' +
                    'allowed={0} readlen={1}'.format(length, len(data)))
        if not data:
            log.msg("EOF: callRemote(close)")
            return True

        if self.remaining is not None:
            self.remaining = self.remaining - len(data)
            assert self.remaining >= 0
        d = self.writer.callRemote('write', data)
        d.addCallback(lambda res: False)
        return d


Instruction:
Write a Python 3 script to
read a block of data from the remote reader.

Response:
def _readBlock(self):
        """Read a block of data from the remote reader."""

        if self.interrupted or self.fp is None:
            if self.debug:
                log.msg('WorkerFileDownloadCommand._readBlock(): end')
            return True

        length = self.blocksize
        if self.bytes_remaining is not None and length > self.bytes_remaining:
            length = self.bytes_remaining

        if length <= 0:
            if self.stderr is None:
                self.stderr = "Maximum filesize reached, truncating file '{0}'".format(
                    self.path)
                self.rc = 1
            return True
        else:
            d = self.reader.callRemote('read', length)
            d.addCallback(self._writeData)
            return d


Instruction:
Can you generate the documentation for the following Python 3 function
def formatMessageForBuildResults(self, mode, buildername, buildset, build, master, previous_results, blamelist):
        ss_list = buildset['sourcestamps']
        results = build['results']

        ctx = dict(results=build['results'],
                   mode=mode,
                   buildername=buildername,
                   workername=build['properties'].get(
                       'workername', ["<unknown>"])[0],
                   buildset=buildset,
                   build=build,
                   projects=self.getProjects(ss_list, master),
                   previous_results=previous_results,
                   status_detected=self.getDetectedStatus(
                       mode, results, previous_results),
                   build_url=utils.getURLForBuild(
                       master, build['builder']['builderid'], build['number']),
                   buildbot_url=master.config.buildbotURL,
                   blamelist=blamelist,
                   summary=self.messageSummary(build, results),
                   sourcestamps=self.messageSourceStamps(ss_list)
                   )
        yield self.buildAdditionalContext(master, ctx)
        msgdict = self.renderMessage(ctx)
        return msgdict

Response:
Generate a buildbot mail message and return a dictionary
           containing the message body type and subject.


Instruction:
Explain what the following Python 3 code does
def fullName(self):
        # join with '_' if both are set (cannot put '.', because it is used as
        # **kwargs)
        if self.parentName and self.name:
            return self.parentName + '_' + self.name
        # otherwise just use the one that is set
        # (this allows empty name for "anonymous nests")
        return self.name or self.parentName

Response: A full name for the current object.


Instruction:
Explain what the following Python 3 code does
def getFromKwargs(self, kwargs):
        args = kwargs.get(self.fullName, [])

        # delete white space for args
        for arg in args:
            if isinstance(arg, str) and not arg.strip():
                args.remove(arg)

        if not args:
            if self.required:
                raise ValidationError(
                    "'%s' needs to be specified" % (self.label))
            if self.multiple:
                args = self.default
            else:
                args = [self.default]

        if self.regex:
            for arg in args:
                if not self.regex.match(arg):
                    raise ValidationError("%s:'%s' does not match pattern '%s'"
                                          % (self.label, arg, self.regex.pattern))
        if self.maxsize is not None:
            for arg in args:
                if len(arg) > self.maxsize:
                    raise ValidationError("%s: is too large %d > %d"
                                          % (self.label, len(arg), self.maxsize))

        try:
            arg = self.parse_from_args(args)
        except Exception as e:
            # an exception will just display an alert in the web UI
            # also log the exception
            if self.debug:
                traceback.print_exc()
            raise e
        if arg is None:
            raise ValidationError("need %s: no default provided by config"
                                  % (self.fullName,))
        return arg

Response:
Simple customization point for child classes that do not need the other
           parameters supplied to updateFromKwargs.


Instruction:
Here you have a function in Python 3, explain what it does
def updateFromKwargs(self, properties, kwargs, collector, **unused):
        properties[self.name] = self.getFromKwargs(kwargs)

Response: Update the properties dictionary with the values from the kwargs.


Instruction:
Given the following Python 3 function, write the documentation
def parse_from_args(self, l):
        if self.multiple:
            return [self.parse_from_arg(arg) for arg in l]
        return self.parse_from_arg(l[0])

Response:
Secondary customization point called from getFromKwargs to turn
           a validated value into a single property value


Instruction:
Create a Python 3 function to
collapse the child values into a dictionary. This is intended to be called by child classes to fix up the fullName - > name conversions.

Response:
def collectChildProperties(self, kwargs, properties, collector, **kw):
        """Collapse the child values into a dictionary. This is intended to be
           called by child classes to fix up the fullName->name conversions."""

        childProperties = {}
        for field in self.fields:  # pylint: disable=not-an-iterable
            yield collector.collectValidationErrors(field.fullName,
                                                    field.updateFromKwargs,
                                                    kwargs=kwargs,
                                                    properties=childProperties,
                                                    collector=collector,
                                                    **kw)
        kwargs[self.fullName] = childProperties


Instruction:
Explain what the following Python 3 code does
def updateFromKwargs(self, kwargs, properties, collector, **kw):
        yield self.collectChildProperties(kwargs=kwargs, properties=properties,
                                          collector=collector, **kw)
        # default behavior is to set a property
        #  -- use setdefault+update in order to collapse 'anonymous' nested
        #     parameters correctly
        if self.name:
            d = properties.setdefault(self.name, {})
        else:
            # if there's no name, collapse this nest all the way
            d = properties
        d.update(kwargs[self.fullName])

Response: Update the properties of the object based on the keyword arguments.


Instruction:
Here you have a function in Python 3, explain what it does
def force(self, owner, builderNames=None, builderid=None, **kwargs):
        builderNames = yield self.computeBuilderNames(builderNames, builderid)
        if not builderNames:
            raise KeyError("builderNames not specified or not supported")

        # Currently the validation code expects all kwargs to be lists
        # I don't want to refactor that now so much sure we comply...
        kwargs = dict((k, [v]) if not isinstance(v, list) else (k, v)
                      for k, v in kwargs.items())

        # probably need to clean that out later as the IProperty is already a
        # validation mechanism
        collector = ValidationErrorCollector()
        reason = yield collector.collectValidationErrors(self.reason.fullName,
                                                         self.reason.getFromKwargs, kwargs)
        if owner is None or owner == "anonymous":
            owner = yield collector.collectValidationErrors(self.username.fullName,
                                                            self.username.getFromKwargs, kwargs)

        properties, changeids, sourcestamps = yield self.gatherPropertiesAndChanges(
            collector, **kwargs)

        collector.maybeRaiseCollectedErrors()

        properties.setProperty("reason", reason, "Force Build Form")
        properties.setProperty("owner", owner, "Force Build Form")

        r = self.reasonString % {'owner': owner, 'reason': reason}

        # turn sourcestamps into a list
        for cb, ss in sourcestamps.items():
            ss['codebase'] = cb
        sourcestamps = list(sourcestamps.values())

        # everything is validated, we can create our source stamp, and
        # buildrequest
        res = yield self.addBuildsetForSourceStampsWithDefaults(
            reason=r,
            sourcestamps=sourcestamps,
            properties=properties,
            builderNames=builderNames,
        )

        return res

Response: Create a new buildset for this source stamp.


Instruction:
Explain what the following Python 3 code does
def get(self, secret, *args, **kwargs):
        for provider in self.services:
            value = yield provider.get(secret)
            source_name = provider.__class__.__name__
            if value is not None:
                return SecretDetails(source_name, secret, value)

Response: get secrets from the provider defined in the secret


Instruction:
Implement a Python 3 function for
saving function save function

Response:
def save():
    """
    save function
    """
    results = {}
    cpu_number = 0

    while True:
        try:
            _file = open(
                CPU_PREFIX + 'cpu{}/cpufreq/scaling_governor'.format(cpu_number))
        except:
            break

        governor = _file.read().strip()
        results.setdefault(cpu_number, {})['governor'] = governor

        _file.close()

        try:
            _file = open(
                CPU_PREFIX + 'cpu{}/cpufreq/scaling_cur_freq'.format(cpu_number))
        except:
            break

        results[cpu_number]['freq'] = _file.read().strip()

        _file.close()

        cpu_number += 1

    return results


Instruction:
Create a Python 3 function to
change the neccesary neccesary

Response:
def change(governor, freq=None):
    """
    change function
    """
    cpu_number = 0

    while True:
        try:
            subprocess.check_output([
                "sudo", "bash", "-c",
                "echo {governor} > {CPU_PREFIX}cpu{cpu_number}/cpufreq/scaling_governor"
                .format(governor=governor,
                        CPU_PREFIX=CPU_PREFIX,
                        cpu_number=cpu_number)],
                                    stderr=subprocess.STDOUT)
        except:
            break

        if freq:
            subprocess.check_output([
                "sudo", "bash", "-c",
                "echo {freq} > {CPU_PREFIX}cpu{cpu_number}/cpufreq/scaling_setspeed"
                .format(freq=freq,
                        CPU_PREFIX=CPU_PREFIX,
                        cpu_number=cpu_number)],
                                    stderr=subprocess.STDOUT)

        cpu_number += 1


Instruction:
Write a Python 3 function for
functioning for checking available frequency

Response:
def available_freq():
    """
    function for checking available frequency
    """
    _file = open(CPU_PREFIX + 'cpu0/cpufreq/scaling_available_frequencies')

    freq = [int(_file) for _file in _file.read().strip().split()]

    _file.close()

    return freq


Instruction:
Write a Python 3 function for
dumping function for the

Response:
def dump():
    """
    dump function
    """

    try:
        sensors = subprocess.check_output('sensors').decode('utf-8')

    except (FileNotFoundError, subprocess.CalledProcessError):
        print("Couldn't read CPU temp")

    else:
        cores = []

        for line in sensors.splitlines():
            if line.startswith('Core '):
                core, rest = line.split(':')
                temp = rest.strip().split()[0]
                cores.append((core, temp))

        for core, temp in cores:
            print(core + ':', temp)

    cpu_number = 0

    while True:
        try:
            _file = open(
                CPU_PREFIX + 'cpu{}/cpufreq/scaling_governor'.format(cpu_number))
        except:
            break

        print('Core ' + str(cpu_number) + ':', _file.read().strip(), end=', ')

        _file.close()

        try:
            _file = open(
                CPU_PREFIX + 'cpu{}/cpufreq/scaling_cur_freq'.format(cpu_number))
        except:
            break

        freq = round(int(_file.read()) / 10 ** 6, 2)

        print(freq, 'GHz')

        cpu_number += 1


Instruction:
Can you generate the documentation for the following Python 3 function
def padto8(data):
    length = len(data)
    return data + b'\xdb' * (roundto8(length) - length)

Response: Pads data to the multiplies of 8 bytes.


Instruction:
Write a Python 3 function that can
parse a cookie string into a dict of name value pairs.

Response:
def parse_cookie(cookie):
    """Parse a ``Cookie`` HTTP header into a dict of name/value pairs.
    This function attempts to mimic browser cookie parsing behavior;
    it specifically does not follow any of the cookie-related RFCs
    (because browsers don't either).
    The algorithm used is identical to that used by Django version 1.9.10.
    """
    cookiedict = {}
    for chunk in cookie.split(str(';')):
        if str('=') in chunk:
            key, val = chunk.split(str('='), 1)
        else:
            # Assume an empty name per
            # https://bugzilla.mozilla.org/show_bug.cgi?id=169091
            key, val = str(''), chunk
        key, val = key.strip(), val.strip()
        if key or val:
            # unquote using Python's algorithm.
            cookiedict[key] = unquote_cookie(val)
    return cookiedict


Instruction:
Explain what the following Python 3 code does
def get_rejected_variables(self, threshold=0.9):
        variable_profile = self.description_set['variables']
        result = []
        if hasattr(variable_profile, 'correlation'):
            result = variable_profile.index[variable_profile.correlation > threshold].tolist()
        return  result

Response:
Return a list of variable names being rejected for high
            correlation with one of remaining variables.


Instruction:
Create a Python 3 function to
write the report to a file.

Response:
def to_file(self, outputfile=DEFAULT_OUTPUTFILE):
        """Write the report to a file.

        By default a name is generated.

        Parameters:
        ----------
        outputfile : str
            The name or the path of the file to generale including the extension (.html).
        """

        if outputfile != NO_OUTPUTFILE:
            if outputfile == DEFAULT_OUTPUTFILE:
                outputfile = 'profile_' + str(hash(self)) + ".html"
            # TODO: should be done in the template
            with codecs.open(outputfile, 'w+b', encoding='utf8') as self.file:
                self.file.write(templates.template('wrapper').render(content=self.html))


Instruction:
Create a Python 3 function to
return a Jinja template ready for rendering.

Response:
def template(template_name):
    """Return a jinja template ready for rendering. If needed, global variables are initialized.

    Parameters
    ----------
    template_name: str, the name of the template as defined in the templates mapping

    Returns
    -------
    The Jinja template ready for rendering
    """
    globals = None
    if template_name.startswith('row_'):
        # This is a row template setting global variable
        globals = dict()
        globals['vartype'] = var_type[template_name.split('_')[1].upper()]
    return jinja2_env.get_template(templates[template_name], globals=globals)


Instruction:
Can you generate the documentation for the following Python 3 function
def to_html(sample, stats_object):

    n_obs = stats_object['table']['n']

    value_formatters = formatters.value_formatters
    row_formatters = formatters.row_formatters

    if not isinstance(sample, pd.DataFrame):
        raise TypeError("sample must be of type pandas.DataFrame")

    if not isinstance(stats_object, dict):
        raise TypeError("stats_object must be of type dict. Did you generate this using the pandas_profiling.describe() function?")

    if not set({'table', 'variables', 'freq', 'correlations'}).issubset(set(stats_object.keys())):
        raise TypeError(
            "stats_object badly formatted. Did you generate this using the pandas_profiling.describe() function?")

    def fmt(value, name):
        if pd.isnull(value):
            return ""
        if name in value_formatters:
            return value_formatters[name](value)
        elif isinstance(value, float):
            return value_formatters[formatters.DEFAULT_FLOAT_FORMATTER](value)
        else:
            try:
                return unicode(value)  # Python 2
            except NameError:
                return str(value)      # Python 3

    def _format_row(freq, label, max_freq, row_template, n, extra_class=''):
            if max_freq != 0:
                width = int(freq / max_freq * 99) + 1
            else:
                width = 1

            if width > 20:
                label_in_bar = freq
                label_after_bar = ""
            else:
                label_in_bar = " "
                label_after_bar = freq

            return row_template.render(label=label,
                                       width=width,
                                       count=freq,
                                       percentage='{:2.1f}'.format(freq / n * 100),
                                       extra_class=extra_class,
                                       label_in_bar=label_in_bar,
                                       label_after_bar=label_after_bar)

    def freq_table(freqtable, n, table_template, row_template, max_number_to_print, nb_col=6):

        freq_rows_html = u''

        if max_number_to_print > n:
                max_number_to_print=n

        if max_number_to_print < len(freqtable):
            freq_other = sum(freqtable.iloc[max_number_to_print:])
            min_freq = freqtable.values[max_number_to_print]
        else:
            freq_other = 0
            min_freq = 0

        freq_missing = n - sum(freqtable)
        max_freq = max(freqtable.values[0], freq_other, freq_missing)

        # TODO: Correctly sort missing and other

        for label, freq in six.iteritems(freqtable.iloc[0:max_number_to_print]):
            freq_rows_html += _format_row(freq, label, max_freq, row_template, n)

        if freq_other > min_freq:
            freq_rows_html += _format_row(freq_other,
                                         "Other values (%s)" % (freqtable.count() - max_number_to_print), max_freq, row_template, n,
                                         extra_class='other')

        if freq_missing > min_freq:
            freq_rows_html += _format_row(freq_missing, "(Missing)", max_freq, row_template, n, extra_class='missing')

        return table_template.render(rows=freq_rows_html, varid=hash(idx), nb_col=nb_col)

    def extreme_obs_table(freqtable, table_template, row_template, number_to_print, n, ascending = True):

        # If it's mixed between base types (str, int) convert to str. Pure "mixed" types are filtered during type discovery
        if "mixed" in freqtable.index.inferred_type:
            freqtable.index = freqtable.index.astype(str)

        sorted_freqTable = freqtable.sort_index()

        if ascending:
            obs_to_print = sorted_freqTable.iloc[:number_to_print]
        else:
            obs_to_print = sorted_freqTable.iloc[-number_to_print:]

        freq_rows_html = ''
        max_freq = max(obs_to_print.values)

        for label, freq in six.iteritems(obs_to_print):
            freq_rows_html += _format_row(freq, label, max_freq, row_template, n)

        return table_template.render(rows=freq_rows_html)

    # Variables
    rows_html = u""
    messages = []
    render_htmls = {}

    for idx, row in stats_object['variables'].iterrows():

        formatted_values = {'varname': idx, 'varid': hash(idx)}
        row_classes = {}

        for col, value in six.iteritems(row):
            formatted_values[col] = fmt(value, col)

        for col in set(row.index) & six.viewkeys(row_formatters):
            row_classes[col] = row_formatters[col](row[col])
            if row_classes[col] == "alert" and col in templates.messages:
                messages.append(templates.messages[col].format(formatted_values, varname = idx))

        if row['type'] in {'CAT', 'BOOL'}:
            formatted_values['minifreqtable'] = freq_table(stats_object['freq'][idx], n_obs,
                                                           templates.template('mini_freq_table'),
                                                           templates.template('mini_freq_table_row'),
                                                           3,
                                                           templates.mini_freq_table_nb_col[row['type']])

            if row['distinct_count'] > 50:
                messages.append(templates.messages['HIGH_CARDINALITY'].format(formatted_values, varname = idx))
                row_classes['distinct_count'] = "alert"
            else:
                row_classes['distinct_count'] = ""

        if row['type'] == 'UNIQUE':
            obs = stats_object['freq'][idx].index

            formatted_values['firstn'] = pd.DataFrame(obs[0:3], columns=["First 3 values"]).to_html(classes="example_values", index=False)
            formatted_values['lastn'] = pd.DataFrame(obs[-3:], columns=["Last 3 values"]).to_html(classes="example_values", index=False)
        if row['type'] == 'UNSUPPORTED':
            formatted_values['varname'] = idx
            messages.append(templates.messages[row['type']].format(formatted_values))
        elif row['type'] in {'CORR', 'CONST', 'RECODED'}:
            formatted_values['varname'] = idx
            messages.append(templates.messages[row['type']].format(formatted_values))
        else:
            formatted_values['freqtable'] = freq_table(stats_object['freq'][idx], n_obs,
                                                       templates.template('freq_table'), templates.template('freq_table_row'), 10)
            formatted_values['firstn_expanded'] = extreme_obs_table(stats_object['freq'][idx], templates.template('freq_table'), templates.template('freq_table_row'), 5, n_obs, ascending = True)
            formatted_values['lastn_expanded'] = extreme_obs_table(stats_object['freq'][idx], templates.template('freq_table'), templates.template('freq_table_row'), 5, n_obs, ascending = False)

        rows_html += templates.row_templates_dict[row['type']].render(values=formatted_values, row_classes=row_classes)
    render_htmls['rows_html'] = rows_html

    # Overview
    formatted_values = {k: fmt(v, k) for k, v in six.iteritems(stats_object['table'])}

    row_classes={}
    for col in six.viewkeys(stats_object['table']) & six.viewkeys(row_formatters):
        row_classes[col] = row_formatters[col](stats_object['table'][col])
        if row_classes[col] == "alert" and col in templates.messages:
            messages.append(templates.messages[col].format(formatted_values, varname = idx))

    messages_html = u''
    for msg in messages:
        messages_html += templates.message_row.format(message=msg)

    overview_html = templates.template('overview').render(values=formatted_values, row_classes = row_classes, messages=messages_html)
    render_htmls['overview_html'] = overview_html

    # Add plot of matrix correlation if the dataframe is not empty
    if len(stats_object['correlations']['pearson']) > 0:
        pearson_matrix = plot.correlation_matrix(stats_object['correlations']['pearson'], 'Pearson')
        spearman_matrix = plot.correlation_matrix(stats_object['correlations']['spearman'], 'Spearman')
        correlations_html = templates.template('correlations').render(
            values={'pearson_matrix': pearson_matrix, 'spearman_matrix': spearman_matrix})
        render_htmls['correlations_html'] = correlations_html

    # Add sample
    sample_html = templates.template('sample').render(sample_table_html=sample.to_html(classes="sample"))
    render_htmls['sample_html'] = sample_html

    # TODO: should be done in the template
    return templates.template('base').render(render_htmls)

Response: Generate a HTML report from summary statistics and a given sample.


Instruction:
Can you implement a function in Python 3 that
computes summary statistics of a numeric variable.

Response:
def describe_numeric_1d(series, **kwargs):
    """Compute summary statistics of a numerical (`TYPE_NUM`) variable (a Series).

    Also create histograms (mini an full) of its distribution.

    Parameters
    ----------
    series : Series
        The variable to describe.

    Returns
    -------
    Series
        The description of the variable as a Series with index being stats keys.
    """
    # Format a number as a percentage. For example 0.25 will be turned to 25%.
    _percentile_format = "{:.0%}"
    stats = dict()
    stats['type'] = base.TYPE_NUM
    stats['mean'] = series.mean()
    stats['std'] = series.std()
    stats['variance'] = series.var()
    stats['min'] = series.min()
    stats['max'] = series.max()
    stats['range'] = stats['max'] - stats['min']
    # To avoid to compute it several times
    _series_no_na = series.dropna()
    for percentile in np.array([0.05, 0.25, 0.5, 0.75, 0.95]):
        # The dropna() is a workaround for https://github.com/pydata/pandas/issues/13098
        stats[_percentile_format.format(percentile)] = _series_no_na.quantile(percentile)
    stats['iqr'] = stats['75%'] - stats['25%']
    stats['kurtosis'] = series.kurt()
    stats['skewness'] = series.skew()
    stats['sum'] = series.sum()
    stats['mad'] = series.mad()
    stats['cv'] = stats['std'] / stats['mean'] if stats['mean'] else np.NaN
    stats['n_zeros'] = (len(series) - np.count_nonzero(series))
    stats['p_zeros'] = stats['n_zeros'] * 1.0 / len(series)
    # Histograms
    stats['histogram'] = histogram(series, **kwargs)
    stats['mini_histogram'] = mini_histogram(series, **kwargs)
    return pd.Series(stats, name=series.name)


Instruction:
How would you code a function in Python 3 to
compute summary statistics of a date variable.

Response:
def describe_date_1d(series):
    """Compute summary statistics of a date (`TYPE_DATE`) variable (a Series).

    Also create histograms (mini an full) of its distribution.

    Parameters
    ----------
    series : Series
        The variable to describe.

    Returns
    -------
    Series
        The description of the variable as a Series with index being stats keys.
    """
    stats = dict()
    stats['type'] = base.TYPE_DATE
    stats['min'] = series.min()
    stats['max'] = series.max()
    stats['range'] = stats['max'] - stats['min']
    # Histograms
    stats['histogram'] = histogram(series)
    stats['mini_histogram'] = mini_histogram(series)
    return pd.Series(stats, name=series.name)


Instruction:
Write a Python 3 function for
computing summary statistics of a categorical variable.

Response:
def describe_categorical_1d(series):
    """Compute summary statistics of a categorical (`TYPE_CAT`) variable (a Series).

    Parameters
    ----------
    series : Series
        The variable to describe.

    Returns
    -------
    Series
        The description of the variable as a Series with index being stats keys.
    """
    # Only run if at least 1 non-missing value
    value_counts, distinct_count = base.get_groupby_statistic(series)
    top, freq = value_counts.index[0], value_counts.iloc[0]
    names = []
    result = []

    if base.get_vartype(series) == base.TYPE_CAT:
        names += ['top', 'freq', 'type']
        result += [top, freq, base.TYPE_CAT]

    return pd.Series(result, index=names, name=series.name)


Instruction:
Can you write a function in Python 3 where it
computes summary statistics of a boolean variable.

Response:
def describe_boolean_1d(series):
    """Compute summary statistics of a boolean (`TYPE_BOOL`) variable (a Series).

    Parameters
    ----------
    series : Series
        The variable to describe.

    Returns
    -------
    Series
        The description of the variable as a Series with index being stats keys.
    """
    value_counts, distinct_count = base.get_groupby_statistic(series)
    top, freq = value_counts.index[0], value_counts.iloc[0]
    # The mean of boolean is an interesting information
    mean = series.mean()
    names = []
    result = []
    names += ['top', 'freq', 'type', 'mean']
    result += [top, freq, base.TYPE_BOOL, mean]

    return pd.Series(result, index=names, name=series.name)


Instruction:
Implement a Python 3 function for
computing summary statistics of a constant variable.

Response:
def describe_constant_1d(series):
    """Compute summary statistics of a constant (`S_TYPE_CONST`) variable (a Series).

    Parameters
    ----------
    series : Series
        The variable to describe.

    Returns
    -------
    Series
        The description of the variable as a Series with index being stats keys.
    """
    return pd.Series([base.S_TYPE_CONST], index=['type'], name=series.name)


Instruction:
Can you generate the documentation for the following Python 3 function
def describe_unique_1d(series):
    return pd.Series([base.S_TYPE_UNIQUE], index=['type'], name=series.name)

Response: Compute summary statistics of a unique variable.


Instruction:
Here you have a function in Python 3, explain what it does
def describe_supported(series, **kwargs):
    leng = len(series)  # number of observations in the Series
    count = series.count()  # number of non-NaN observations in the Series
    n_infinite = count - series.count()  # number of infinte observations in the Series

    value_counts, distinct_count = base.get_groupby_statistic(series)
    if count > distinct_count > 1:
        mode = series.mode().iloc[0]
    else:
        mode = series[0]

    results_data = {'count': count,
                    'distinct_count': distinct_count,
                    'p_missing': 1 - count * 1.0 / leng,
                    'n_missing': leng - count,
                    'p_infinite': n_infinite * 1.0 / leng,
                    'n_infinite': n_infinite,
                    'is_unique': distinct_count == leng,
                    'mode': mode,
                    'p_unique': distinct_count * 1.0 / leng}
    try:
        # pandas 0.17 onwards
        results_data['memorysize'] = series.memory_usage()
    except:
        results_data['memorysize'] = 0

    return pd.Series(results_data, name=series.name)

Response: Compute summary statistics of a supported variable.


Instruction:
Explain what the following Python 3 code does
def describe_unsupported(series, **kwargs):
    leng = len(series)  # number of observations in the Series
    count = series.count()  # number of non-NaN observations in the Series
    n_infinite = count - series.count()  # number of infinte observations in the Series

    results_data = {'count': count,
                    'p_missing': 1 - count * 1.0 / leng,
                    'n_missing': leng - count,
                    'p_infinite': n_infinite * 1.0 / leng,
                    'n_infinite': n_infinite,
                    'type': base.S_TYPE_UNSUPPORTED}

    try:
        # pandas 0.17 onwards
        results_data['memorysize'] = series.memory_usage()
    except:
        results_data['memorysize'] = 0

    return pd.Series(results_data, name=series.name)

Response: Compute summary statistics of an unsupported variable.


Instruction:
Create a Python 3 function to
compute summary statistics of a single 1D variable.

Response:
def describe_1d(data, **kwargs):
    """Compute summary statistics of a variable (a Series).

    The description is different according to the type of the variable.
    However a set of common stats is also computed.

    Parameters
    ----------
    series : Series
        The variable to describe.

    Returns
    -------
    Series
        The description of the variable as a Series with index being stats keys.
    """

    # Replace infinite values with NaNs to avoid issues with
    # histograms later.
    data.replace(to_replace=[np.inf, np.NINF, np.PINF], value=np.nan, inplace=True)

    result = pd.Series({}, name=data.name)

    vartype = base.get_vartype(data)

    if vartype == base.S_TYPE_UNSUPPORTED:
        result = result.append(describe_unsupported(data))
    else:
        result = result.append(describe_supported(data))

        if vartype == base.S_TYPE_CONST:
            result = result.append(describe_constant_1d(data))
        elif vartype == base.TYPE_BOOL:
            result = result.append(describe_boolean_1d(data))
        elif vartype == base.TYPE_NUM:
            result = result.append(describe_numeric_1d(data, **kwargs))
        elif vartype == base.TYPE_DATE:
            result = result.append(describe_date_1d(data))
        elif vartype == base.S_TYPE_UNIQUE:
            result = result.append(describe_unique_1d(data))
        else:
            # TYPE_CAT
            result = result.append(describe_categorical_1d(data))

    return result


Instruction:
Can you create a Python 3 function that
generates a dict containing summary statistics for a given dataset stored as a pandas. DataFrame.

Response:
def describe(df, bins=10, check_correlation=True, correlation_threshold=0.9, correlation_overrides=None, check_recoded=False, pool_size=multiprocessing.cpu_count(), **kwargs):
    """Generates a dict containing summary statistics for a given dataset stored as a pandas `DataFrame`.

    Used has is it will output its content as an HTML report in a Jupyter notebook.

    Parameters
    ----------
    df : DataFrame
        Data to be analyzed
    bins : int
        Number of bins in histogram.
        The default is 10.
    check_correlation : boolean
        Whether or not to check correlation.
        It's `True` by default.
    correlation_threshold: float
        Threshold to determine if the variable pair is correlated.
        The default is 0.9.
    correlation_overrides : list
        Variable names not to be rejected because they are correlated.
        There is no variable in the list (`None`) by default.
    check_recoded : boolean
        Whether or not to check recoded correlation (memory heavy feature).
        Since it's an expensive computation it can be activated for small datasets.
        `check_correlation` must be true to disable this check.
        It's `False` by default.
    pool_size : int
        Number of workers in thread pool
        The default is equal to the number of CPU.

    Returns
    -------
    dict
        Containing the following keys:
            * table: general statistics on the dataset
            * variables: summary statistics for each variable
            * freq: frequency table

    Notes:
    ------
        * The section dedicated to check the correlation should be externalized
    """

    if not isinstance(df, pd.DataFrame):
        raise TypeError("df must be of type pandas.DataFrame")
    if df.empty:
        raise ValueError("df can not be empty")

    try:
        # reset matplotlib style before use
        # Fails in matplotlib 1.4.x so plot might look bad
        matplotlib.style.use("default")
    except:
        pass

    try:
        # Ignore FutureWarning
        from pandas.plotting import register_matplotlib_converters
        register_matplotlib_converters()
    except:
        pass

    matplotlib.style.use(resource_filename(__name__, "pandas_profiling.mplstyle"))

    # Clearing the cache before computing stats
    base.clear_cache()

    if not pd.Index(np.arange(0, len(df))).equals(df.index):
        # Treat index as any other column
        df = df.reset_index()

    kwargs.update({'bins': bins})
    # Describe all variables in a univariate way
    if pool_size == 1:
        local_multiprocess_func = partial(multiprocess_func, **kwargs)
        ldesc = {col: s for col, s in map(local_multiprocess_func, df.iteritems())}
    else:
        pool = multiprocessing.Pool(pool_size)
        local_multiprocess_func = partial(multiprocess_func, **kwargs)
        ldesc = {col: s for col, s in pool.map(local_multiprocess_func, df.iteritems())}
        pool.close()

    # Get correlations
    dfcorrPear = df.corr(method="pearson")
    dfcorrSpear = df.corr(method="spearman")

    # Check correlations between variable
    if check_correlation is True:
        ''' TODO: corr(x,y) > 0.9 and corr(y,z) > 0.9 does not imply corr(x,z) > 0.9
        If x~y and y~z but not x~z, it would be better to delete only y
        Better way would be to find out which variable causes the highest increase in multicollinearity.
        '''
        corr = dfcorrPear.copy()
        for x, corr_x in corr.iterrows():
            if correlation_overrides and x in correlation_overrides:
                continue

            for y, corr in corr_x.iteritems():
                if x == y: break

                if corr > correlation_threshold:
                    ldesc[x] = pd.Series(['CORR', y, corr], index=['type', 'correlation_var', 'correlation'])

        if check_recoded:
            categorical_variables = [(name, data) for (name, data) in df.iteritems() if base.get_vartype(data)=='CAT']
            for (name1, data1), (name2, data2) in itertools.combinations(categorical_variables, 2):
                if correlation_overrides and name1 in correlation_overrides:
                    continue

                confusion_matrix=pd.crosstab(data1,data2)
                if confusion_matrix.values.diagonal().sum() == len(df):
                    ldesc[name1] = pd.Series(['RECODED', name2], index=['type', 'correlation_var'])

    # Convert ldesc to a DataFrame
    names = []
    ldesc_indexes = sorted([x.index for x in ldesc.values()], key=len)
    for idxnames in ldesc_indexes:
        for name in idxnames:
            if name not in names:
                names.append(name)
    variable_stats = pd.concat(ldesc, join_axes=pd.Index([names]), axis=1)
    variable_stats.columns.names = df.columns.names

    # General statistics
    table_stats = {}

    table_stats['n'] = len(df)
    table_stats['nvar'] = len(df.columns)
    table_stats['total_missing'] = variable_stats.loc['n_missing'].sum() / (table_stats['n'] * table_stats['nvar'])
    unsupported_columns = variable_stats.transpose()[variable_stats.transpose().type != base.S_TYPE_UNSUPPORTED].index.tolist()
    table_stats['n_duplicates'] = sum(df.duplicated(subset=unsupported_columns)) if len(unsupported_columns) > 0 else 0

    memsize = df.memory_usage(index=True).sum()
    table_stats['memsize'] = formatters.fmt_bytesize(memsize)
    table_stats['recordsize'] = formatters.fmt_bytesize(memsize / table_stats['n'])

    table_stats.update({k: 0 for k in ("NUM", "DATE", "CONST", "CAT", "UNIQUE", "CORR", "RECODED", "BOOL", "UNSUPPORTED")})
    table_stats.update(dict(variable_stats.loc['type'].value_counts()))
    table_stats['REJECTED'] = table_stats['CONST'] + table_stats['CORR'] + table_stats['RECODED']

    return {
        'table': table_stats,
        'variables': variable_stats.T,
        'freq': {k: (base.get_groupby_statistic(df[k])[0] if variable_stats[k].type != base.S_TYPE_UNSUPPORTED else None) for k in df.columns},
        'correlations': {'pearson': dfcorrPear, 'spearman': dfcorrSpear}
    }


Instruction:
Given the following Python 3 function, write the documentation
def _plot_histogram(series, bins=10, figsize=(6, 4), facecolor='#337ab7'):
    if base.get_vartype(series) == base.TYPE_DATE:
        # TODO: These calls should be merged
        fig = plt.figure(figsize=figsize)
        plot = fig.add_subplot(111)
        plot.set_ylabel('Frequency')
        try:
            plot.hist(series.dropna().values, facecolor=facecolor, bins=bins)
        except TypeError: # matplotlib 1.4 can't plot dates so will show empty plot instead
            pass
    else:
        plot = series.plot(kind='hist', figsize=figsize,
                           facecolor=facecolor,
                           bins=bins)  # TODO when running on server, send this off to a different thread
    return plot

Response: Plot a histogram from the data and return the AxesSubplot object.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def histogram(series, **kwargs):
    imgdata = BytesIO()
    plot = _plot_histogram(series, **kwargs)
    plot.figure.subplots_adjust(left=0.15, right=0.95, top=0.9, bottom=0.1, wspace=0, hspace=0)
    plot.figure.savefig(imgdata)
    imgdata.seek(0)
    result_string = 'data:image/png;base64,' + quote(base64.b64encode(imgdata.getvalue()))
    # TODO Think about writing this to disk instead of caching them in strings
    plt.close(plot.figure)
    return result_string

Response: Plot an histogram of the data.


Instruction:
How would you code a function in Python 3 to
plot a small histogram of the data.

Response:
def mini_histogram(series, **kwargs):
    """Plot a small (mini) histogram of the data.

    Parameters
    ----------
    series: Series
        The data to plot.

    Returns
    -------
    str
        The resulting image encoded as a string.
    """
    imgdata = BytesIO()
    plot = _plot_histogram(series, figsize=(2, 0.75), **kwargs)
    plot.axes.get_yaxis().set_visible(False)

    if LooseVersion(matplotlib.__version__) <= '1.5.9':
        plot.set_axis_bgcolor("w")
    else:
        plot.set_facecolor("w")

    xticks = plot.xaxis.get_major_ticks()
    for tick in xticks[1:-1]:
        tick.set_visible(False)
        tick.label.set_visible(False)
    for tick in (xticks[0], xticks[-1]):
        tick.label.set_fontsize(8)
    plot.figure.subplots_adjust(left=0.15, right=0.85, top=1, bottom=0.35, wspace=0, hspace=0)
    plot.figure.savefig(imgdata)
    imgdata.seek(0)
    result_string = 'data:image/png;base64,' + quote(base64.b64encode(imgdata.getvalue()))
    plt.close(plot.figure)
    return result_string


Instruction:
Can you tell what is the following Python 3 function doing
def correlation_matrix(corrdf, title, **kwargs):
    imgdata = BytesIO()
    fig_cor, axes_cor = plt.subplots(1, 1)
    labels = corrdf.columns
    matrix_image = axes_cor.imshow(corrdf, vmin=-1, vmax=1, interpolation="nearest", cmap='bwr')
    plt.title(title, size=18)
    plt.colorbar(matrix_image)
    axes_cor.set_xticks(np.arange(0, corrdf.shape[0], corrdf.shape[0] * 1.0 / len(labels)))
    axes_cor.set_yticks(np.arange(0, corrdf.shape[1], corrdf.shape[1] * 1.0 / len(labels)))
    axes_cor.set_xticklabels(labels, rotation=90)
    axes_cor.set_yticklabels(labels)

    matrix_image.figure.savefig(imgdata, bbox_inches='tight')
    imgdata.seek(0)
    result_string = 'data:image/png;base64,' + quote(base64.b64encode(imgdata.getvalue()))
    plt.close(matrix_image.figure)
    return result_string

Response: Plot image of a matrix correlation.


Instruction:
Can you write a function in Python 3 where it
calculates value counts and distinct count of a variable.

Response:
def get_groupby_statistic(data):
    """Calculate value counts and distinct count of a variable (technically a Series).

    The result is cached by column name in a global variable to avoid recomputing.

    Parameters
    ----------
    data : Series
        The data type of the Series.

    Returns
    -------
    list
        value count and distinct count
    """
    if data.name is not None and data.name in _VALUE_COUNTS_MEMO:
        return _VALUE_COUNTS_MEMO[data.name]

    value_counts_with_nan = data.value_counts(dropna=False)
    value_counts_without_nan = value_counts_with_nan.reset_index().dropna().set_index('index').iloc[:,0]
    distinct_count_with_nan = value_counts_with_nan.count()

    # When the inferred type of the index is just "mixed" probably the types within the series are tuple, dict, list and so on...
    if value_counts_without_nan.index.inferred_type == "mixed":
        raise TypeError('Not supported mixed type')

    result = [value_counts_without_nan, distinct_count_with_nan]

    if data.name is not None:
        _VALUE_COUNTS_MEMO[data.name] = result

    return result


Instruction:
Implement a function in Python 3 to
infer the type of a variable.

Response:
def get_vartype(data):
    """Infer the type of a variable (technically a Series).

    The types supported are split in standard types and special types.

    Standard types:
        * Categorical (`TYPE_CAT`): the default type if no other one can be determined
        * Numerical (`TYPE_NUM`): if it contains numbers
        * Boolean (`TYPE_BOOL`): at this time only detected if it contains boolean values, see todo
        * Date (`TYPE_DATE`): if it contains datetime

    Special types:
        * Constant (`S_TYPE_CONST`): if all values in the variable are equal
        * Unique (`S_TYPE_UNIQUE`): if all values in the variable are different
        * Unsupported (`S_TYPE_UNSUPPORTED`): if the variable is unsupported

     The result is cached by column name in a global variable to avoid recomputing.

    Parameters
    ----------
    data : Series
        The data type of the Series.

    Returns
    -------
    str
        The data type of the Series.

    Notes
    ----
        * Should improve verification when a categorical or numeric field has 3 values, it could be a categorical field
        or just a boolean with NaN values
        * #72: Numeric with low Distinct count should be treated as "Categorical"
    """
    if data.name is not None and data.name in _MEMO:
        return _MEMO[data.name]

    vartype = None
    try:
        distinct_count = get_groupby_statistic(data)[1]
        leng = len(data)

        if distinct_count <= 1:
            vartype = S_TYPE_CONST
        elif pd.api.types.is_bool_dtype(data) or (distinct_count == 2 and pd.api.types.is_numeric_dtype(data)):
            vartype = TYPE_BOOL
        elif pd.api.types.is_numeric_dtype(data):
            vartype = TYPE_NUM
        elif pd.api.types.is_datetime64_dtype(data):
            vartype = TYPE_DATE
        elif distinct_count == leng:
            vartype = S_TYPE_UNIQUE
        else:
            vartype = TYPE_CAT
    except:
        vartype = S_TYPE_UNSUPPORTED

    if data.name is not None:
        _MEMO[data.name] = vartype

    return vartype


Instruction:
How would you code a function in Python 3 to
determine the immediate parent boolean operator for a filter

Response:
def get_block_operator(self):
        """Determine the immediate parent boolean operator for a filter"""
        # Top level operator is `and`
        block_stack = ['and']
        for f in self.manager.iter_filters(block_end=True):
            if f is None:
                block_stack.pop()
                continue
            if f.type in ('and', 'or', 'not'):
                block_stack.append(f.type)
            if f == self:
                break
        return block_stack[-1]


Instruction:
Can you tell what is the following Python 3 function doing
def _validate_resource_count(self):
        for field in ('op', 'value'):
            if field not in self.data:
                raise PolicyValidationError(
                    "Missing '%s' in value filter %s" % (field, self.data))

        if not (isinstance(self.data['value'], int) or
                isinstance(self.data['value'], list)):
            raise PolicyValidationError(
                "`value` must be an integer in resource_count filter %s" % self.data)

        # I don't see how to support regex for this?
        if self.data['op'] not in OPERATORS or self.data['op'] in {'regex', 'regex-case'}:
            raise PolicyValidationError(
                "Invalid operator in value filter %s" % self.data)

        return self

Response: Specific validation for resource_count type


Instruction:
Make a summary of the following Python 3 code
def load_manifest_file(client, bucket, schema, versioned, ifilters, key_info):
    # To avoid thundering herd downloads, we do an immediate yield for
    # interspersed i/o
    yield None

    # Inline these values to avoid the local var lookup, they are constants
    # rKey = schema['Key'] # 1
    # rIsLatest = schema['IsLatest'] # 3
    # rVersionId = schema['VersionId'] # 2

    with tempfile.NamedTemporaryFile() as fh:
        client.download_fileobj(Bucket=bucket, Key=key_info['key'], Fileobj=fh)
        fh.seek(0)
        reader = csv.reader(gzip.GzipFile(fileobj=fh, mode='r'))
        for key_set in chunks(reader, 1000):
            keys = []
            for kr in key_set:
                k = kr[1]
                if inventory_filter(ifilters, schema, kr):
                    continue
                k = unquote_plus(k)
                if versioned:
                    if kr[3] == 'true':
                        keys.append((k, kr[2], True))
                    else:
                        keys.append((k, kr[2]))
                else:
                    keys.append(k)
            yield keys

Response: Given an inventory csv file return an iterator over keys


Instruction:
Can you tell what is the following Python 3 function doing
def load_bucket_inventory(
        client, inventory_bucket, inventory_prefix, versioned, ifilters):
    """Given an inventory location for a bucket, return an iterator over keys

    on the most recent delivered manifest.
    """
    now = datetime.datetime.now()
    key_prefix = "%s/%s" % (inventory_prefix, now.strftime('%Y-%m-'))
    keys = client.list_objects(
        Bucket=inventory_bucket, Prefix=key_prefix).get('Contents', [])
    keys = [k['Key'] for k in keys if k['Key'].endswith('.json')]
    keys.sort()
    if not keys:
        # no manifest delivery
        return None
    latest_manifest = keys[-1]
    manifest = client.get_object(Bucket=inventory_bucket, Key=latest_manifest)
    manifest_data = json.load(manifest['Body'])

    # schema as column name to column index mapping
    schema = dict([(k, i) for i, k in enumerate(
        [n.strip() for n in manifest_data['fileSchema'].split(',')])])

    processor = functools.partial(
        load_manifest_file, client, inventory_bucket,
        schema, versioned, ifilters)
    generators = map(processor, manifest_data.get('files', ()))
    return random_chain(generators)

Response:
Given an inventory location for a bucket return an iterator over keys
    on the most recent delivered manifest.


Instruction:
Can you implement a function in Python 3 that
checks a bucket for a named inventory and return the destination.

Response:
def get_bucket_inventory(client, bucket, inventory_id):
    """Check a bucket for a named inventory, and return the destination."""
    inventories = client.list_bucket_inventory_configurations(
        Bucket=bucket).get('InventoryConfigurationList', [])
    inventories = {i['Id']: i for i in inventories}
    found = fnmatch.filter(inventories, inventory_id)
    if not found:
        return None

    i = inventories[found.pop()]
    s3_info = i['Destination']['S3BucketDestination']
    return {'bucket': s3_info['Bucket'].rsplit(':')[-1],
            'prefix': "%s/%s/%s" % (s3_info['Prefix'], bucket, i['Id'])}


Instruction:
Write a Python 3 function for
creating a new HTML file for the current language.

Response:
def create_html_file(config):
    """ You can customize the automated documentation by altering
        the code directly in this script or the associated jinja2 template
    """
    logging.debug("Starting create_html_file")
    logging.debug(
        "\tjinja2_template_file = {}"
        .format(config['jinja2_template_filename']))
    logging.debug(
        "\ttrendered_filename = {}"
        .format(config['rendered_filename']))

    ts = time.time()
    timestamp = datetime.datetime.utcfromtimestamp(ts).strftime(
        '%Y-%m-%d %H:%M:%S')
    script_path = os.path.dirname(os.path.abspath(__file__))
    rendered_file_path = os.path.join(
        script_path, config['rendered_filename'])
    environment = jinja2.Environment(
        loader=jinja2.FileSystemLoader(script_path))

    environment_column = True if config['environment_tags'] else False

    render_vars = {
        "timestamp": timestamp,
        "c7n_data": c7n_data,
        "environment_column": environment_column,
        "environment_tags": config['environment_tags']
    }

    with open(rendered_file_path, "w") as result_file:
        result_file.write(
            environment.get_template(config['jinja2_template_filename'])
            .render(render_vars))

    logging.debug("File created: %s", rendered_file_path)

    return rendered_file_path


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_file_url(path, config):
    file_url_regex = re.compile(config['file_url_regex'])
    new_path = re.sub(file_url_regex, config['file_url_base'], path)
    return new_path

Response: Update this function to help build the link to your file


Instruction:
Can you generate a brief explanation for the following Python 3 code
def gather_file_data(config):
    file_regex = re.compile(config['file_regex'])
    category_regex = re.compile(config['category_regex'])
    policies = {}

    for root, dirs, files in os.walk(config['c7n_policy_directory']):
        for file in files:
            if file_regex.match(file):
                file_path = root + '/' + file
                logging.debug('Processing file %s', file_path)
                with open(file_path, 'r') as stream:
                    try:
                        if category_regex.search(file_path):
                            category = 'Security & Governance'
                        else:
                            category = 'Cost Controls'

                        policies = yaml.load(stream)
                        for policy in policies['policies']:
                            logging.debug(
                                'Processing policy %s', policy['name'])
                            policy['file_url'] = get_file_url(
                                file_path, config)
                            resource_type = policy['resource']
                            if category not in c7n_data:
                                c7n_data[category] = {}
                            if resource_type not in c7n_data[category]:
                                c7n_data[category][resource_type] = []
                            c7n_data[category][resource_type].append(policy)
                    except yaml.YAMLError as exc:
                        logging.error(exc)

Response: Gather policy information from files


Instruction:
Implement a function in Python 3 to
upload html file to S3

Response:
def upload_to_s3(file_path, config):
    """ Upload html file to S3
    """
    logging.info("Uploading file to S3 bucket: %s", config['s3_bucket_name'])
    s3 = boto3.resource('s3')
    s3_filename = config['s3_bucket_path'] + config['rendered_filename']
    s3.Bucket(config['s3_bucket_name']).upload_file(
        file_path, s3_filename, ExtraArgs={
            'ContentType': 'text/html', 'ACL': 'public-read'})


Instruction:
Can you generate a brief explanation for the following Python 3 code
def github_repos(organization, github_url, github_token):
    # Get github repos
    headers = {"Authorization": "token {}".format(github_token)}
    next_cursor = None

    while next_cursor is not False:
        params = {'query': query, 'variables': {
            'organization': organization, 'cursor': next_cursor}}
        response = requests.post(github_url, headers=headers, json=params)
        result = response.json()
        if response.status_code != 200 or 'errors' in result:
            raise ValueError("Github api error %s" % (
                response.content.decode('utf8'),))

        repos = jmespath.search(
            'data.organization.repositories.edges[].node', result)
        for r in repos:
            yield r
        page_info = jmespath.search(
            'data.organization.repositories.pageInfo', result)
        if page_info:
            next_cursor = (page_info['hasNextPage'] and
                           page_info['endCursor'] or False)
        else:
            next_cursor = False

Response: Return all github repositories in an organization.


Instruction:
Write a Python 3 function that can
stream changes for repos in a GitHub organization.

Response:
def org_stream(ctx, organization, github_url, github_token, clone_dir,
               verbose, filter, exclude, stream_uri, assume):
    """Stream changes for repos in a GitHub organization.
    """
    logging.basicConfig(
        format="%(asctime)s: %(name)s:%(levelname)s %(message)s",
        level=(verbose and logging.DEBUG or logging.INFO))

    log.info("Checkout/Update org repos")
    repos = ctx.invoke(
        org_checkout,
        organization=organization,
        github_url=github_url,
        github_token=github_token,
        clone_dir=clone_dir,
        verbose=verbose,
        filter=filter,
        exclude=exclude)

    log.info('Streaming org changes')
    change_count = 0
    for r in repos:
        change_count += ctx.invoke(
            stream,
            repo_uri=r,
            stream_uri=stream_uri,
            verbose=verbose,
            assume=assume)
    log.info("Streamed %d org changes", change_count)


Instruction:
Explain what the following Python 3 code does
def diff(repo_uri, source, target, output, verbose):
    logging.basicConfig(
        format="%(asctime)s: %(name)s:%(levelname)s %(message)s",
        level=(verbose and logging.DEBUG or logging.INFO))
    logging.getLogger('botocore').setLevel(logging.WARNING)

    if repo_uri is None:
        repo_uri = pygit2.discover_repository(os.getcwd())

    repo = pygit2.Repository(repo_uri)
    load_resources()

    # If on master show diff between last commit to current head
    if repo.head.shorthand == 'master':
        if source is None:
            source = 'master@{1}'
        if target is None:
            target = 'master'
    # Else show difference between master and current head
    elif target is None:
        target = repo.head.shorthand
    if source is None:
        source = 'master'

    policy_repo = PolicyRepo(repo_uri, repo)
    changes = list(policy_repo.delta_commits(
        repo.revparse_single(source), repo.revparse_single(target)))
    output.write(
        yaml.safe_dump({
            'policies': [c.policy.data for c in changes
                         if c.kind != ChangeType.REMOVE]}).encode('utf8'))

Response: Policy diff between two arbitrary revisions.


Instruction:
Create a Python 3 function for
streaming git history policy changes to a local file.

Response:
def stream(repo_uri, stream_uri, verbose, assume, sort, before=None, after=None):
    """Stream git history policy changes to destination.

    Default stream destination is a summary of the policy changes to stdout, one
    per line. Also supported for stdout streaming is `jsonline`.

    AWS Kinesis and SQS destinations are specified by providing the ARN.

    Database destinations are supported by providing a sqlalchemy DSN. Note
    SQLAlchemy and db drivers must be installed separately as they an optional
    dependency.

    When using database destinations, streaming defaults to incremental.
    """
    logging.basicConfig(
        format="%(asctime)s: %(name)s:%(levelname)s %(message)s",
        level=(verbose and logging.DEBUG or logging.INFO))
    logging.getLogger('botocore').setLevel(logging.WARNING)

    if before:
        before = parse(before)
    if after:
        after = parse(after)
    if sort:
        sort = six.moves.reduce(operator.or_, [SORT_TYPE[s] for s in sort])

    with contextlib.closing(TempDir().open()) as temp_dir:
        if repo_uri is None:
            repo_uri = pygit2.discover_repository(os.getcwd())
            log.debug("Using repository %s", repo_uri)
        if repo_uri.startswith('http') or repo_uri.startswith('git@'):
            log.info("Cloning repository: %s", repo_uri)
            repo = pygit2.clone_repository(repo_uri, temp_dir.path)
        else:
            repo = pygit2.Repository(repo_uri)
        load_resources()
        policy_repo = PolicyRepo(repo_uri, repo)
        change_count = 0

        with contextlib.closing(transport(stream_uri, assume)) as t:
            if after is None and isinstance(t, IndexedTransport):
                after = t.last()
            for change in policy_repo.delta_stream(after=after, before=before):
                change_count += 1
                t.send(change)

        log.info("Streamed %d policy repo changes", change_count)
    return change_count


Instruction:
Write a Python 3 script for
returning the named subset of policies

Response:
def select(self, names):
        """return the named subset of policies"""
        return PolicyCollection(
            [p for p in self.policies if p.name in names], self.options)


Instruction:
Here you have a function in Python 3, explain what it does
def delta_commits(self, baseline, target):
        baseline_files = self._get_policy_fents(baseline.tree)
        target_files = self._get_policy_fents(target.tree)

        baseline_policies = PolicyCollection()
        target_policies = PolicyCollection()

        # Added
        for f in set(target_files) - set(baseline_files):
            target_policies += self._policy_file_rev(f, target)

        # Removed
        for f in set(baseline_files) - set(target_files):
            baseline_policies += self._policy_file_rev(f, baseline)

        # Modified
        for f in set(baseline_files).intersection(target_files):
            if baseline_files[f].hex == target_files[f].hex:
                continue
            target_policies += self._policy_file_rev(f, target)
            baseline_policies += self._policy_file_rev(f, baseline)

        return CollectionDelta(
            baseline_policies, target_policies, target, self.repo_uri).delta()

Response: Show policies changes between arbitrary commits.


Instruction:
Create a Python 3 function for
returning an iterator of policy changes along a commit lineage in a repo.

Response:
def delta_stream(self, target='HEAD', limit=None,
                     sort=pygit2.GIT_SORT_TIME | pygit2.GIT_SORT_REVERSE,
                     after=None, before=None):
        """Return an iterator of policy changes along a commit lineage in a repo.
        """
        if target == 'HEAD':
            target = self.repo.head.target

        commits = []
        for commit in self.repo.walk(target, sort):
            cdate = commit_date(commit)
            log.debug(
                "processing commit id:%s date:%s parents:%d msg:%s",
                str(commit.id)[:6], cdate.isoformat(),
                len(commit.parents), commit.message)
            if after and cdate > after:
                continue
            if before and cdate < before:
                continue
            commits.append(commit)
            if limit and len(commits) > limit:
                break

        if limit:
            self.initialize_tree(commits[limit].tree)
            commits.pop(-1)

        for commit in commits:
            for policy_change in self._process_stream_commit(commit):
                yield policy_change


Instruction:
Given the following Python 3 function, write the documentation
def _process_stream_delta(self, delta_stream):
        for pchange in delta_stream:
            if pchange.kind == ChangeType.ADD:
                self.policy_files.setdefault(
                    pchange.file_path, PolicyCollection()).add(pchange.policy)
            elif pchange.kind == ChangeType.REMOVE:
                self.policy_files[pchange.file_path].remove(pchange.policy)
            elif pchange.kind in (ChangeType.MOVED, ChangeType.MODIFIED):
                if pchange.policy.file_path != pchange.previous.file_path:
                    self.policy_files[pchange.previous.file_path].remove(pchange.previous)
                    if (pchange.policy.file_path in self.policy_files and
                            pchange.policy.name in self.policy_files[pchange.file_path]):
                        self.policy_files[pchange.file_path][pchange.policy.name] = pchange.policy
                    else:
                        self.policy_files.setdefault(
                            pchange.file_path, PolicyCollection()).add(pchange.policy)
                else:
                    self.policy_files[pchange.file_path][pchange.policy.name] = pchange.policy
            yield pchange

Response: Iterates over the delta_stream and adds or removes the new entries.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def send(self, change):
        self.buf.append(change)
        if len(self.buf) % self.BUF_SIZE == 0:
            self.flush()

Response: send the given policy change


Instruction:
Can you generate the documentation for the following Python 3 function
def flush(self):
        buf = self.buf
        self.buf = []
        if buf:
            self._flush(buf)

Response: flush any buffered messages


Instruction:
Can you tell what is the following Python 3 function doing
def process_firehose_archive(bucket, key):
    data = {}
    with tempfile.NamedTemporaryFile(mode='w+b') as fh:
        s3.download_file(bucket, key, fh.name)
        log.warning("Downloaded Key Size:%s Key:%s",
                    sizeof_fmt(os.path.getsize(fh.name)), key)
        fh.seek(0, 0)
        record_count = 0
        iteration_count = 0
        for r in records_iter(gzip.GzipFile(fh.name, mode='r')):
            record_count += len(r['logEvents'])
            iteration_count += 1
            key = '%s/%s/%s' % (r['owner'], r['logGroup'], r['logStream'])
            data.setdefault(key, []).extend(r['logEvents'])
            if record_count > EVENTS_SIZE_BUFFER:
                log.warning(
                    "Incremental Data Load records:%d enis:%d",
                    record_count,
                    len(data))
                for k in data:
                    process_record_set(k, data[k])
                data.clear()
                gc.collect()
                record_count = 0

        for k in data:
            process_record_set(k, data[k])
        data.clear()
        gc.collect()

Response: Download firehose archive and aggregate records in memory and write back.


Instruction:
Can you generate the documentation for the following Python 3 function
def records_iter(fh, buffer_size=1024 * 1024 * 16):
    buf = None
    while True:
        chunk = fh.read(buffer_size)
        if not chunk:
            if buf:
                yield json.loads(buf)
            return
        if buf:
            chunk = b"%s%s" % (buf, chunk)
            buf = None
        while chunk:
            idx = chunk.find(b'}{')
            if idx == -1:
                buf = chunk
                chunk = None
                continue
            record = chunk[:idx + 1]
            yield json.loads(record)
            chunk = chunk[idx + 1:]

Response: Iterate over the records in a firehose s3 object.


Instruction:
Write a Python 3 function for
getting an active session in the target account.

Response:
def get_session(self, account_id):
        """Get an active session in the target account."""
        if account_id not in self.account_sessions:
            if account_id not in self.config['accounts']:
                raise AccountNotFound("account:%s is unknown" % account_id)

            self.account_sessions[account_id] = s = assumed_session(
                self.config['accounts'][account_id]['role'], "Sphere11")
            s._session.user_agent_name = "Sphere11"
            s._session.user_agent_version = "0.07"
        return self.account_sessions[account_id]


Instruction:
Implement a function in Python 3 to
scope a schema error to its policy name and resource.

Response:
def policy_error_scope(error, data):
    """Scope a schema error to its policy name and resource."""
    err_path = list(error.absolute_path)
    if err_path[0] != 'policies':
        return error
    pdata = data['policies'][err_path[1]]
    pdata.get('name', 'unknown')
    error.message = "Error on policy:{} resource:{}\n".format(
        pdata.get('name', 'unknown'), pdata.get('resource', 'unknown')) + error.message
    return error


Instruction:
How would you implement a function in Python 3 that
tries to find the best error for humans to resolve

Response:
def specific_error(error):
    """Try to find the best error for humans to resolve

    The jsonschema.exceptions.best_match error is based purely on a
    mix of a strong match (ie. not anyOf, oneOf) and schema depth,
    this often yields odd results that are semantically confusing,
    instead we can use a bit of structural knowledge of schema to
    provide better results.
    """
    if error.validator not in ('anyOf', 'oneOf'):
        return error

    r = t = None

    if isinstance(error.instance, dict):
        t = error.instance.get('type')
        r = error.instance.get('resource')

    if r is not None:
        found = None
        for idx, v in enumerate(error.validator_value):
            if v['$ref'].rsplit('/', 2)[1].endswith(r):
                found = idx
                break
        if found is not None:
            # error context is a flat list of all validation
            # failures, we have to index back to the policy
            # of interest.
            for e in error.context:
                # resource policies have a fixed path from
                # the top of the schema
                if e.absolute_schema_path[4] == found:
                    return specific_error(e)
            return specific_error(error.context[idx])

    if t is not None:
        found = None
        for idx, v in enumerate(error.validator_value):
            if '$ref' in v and v['$ref'].rsplit('/', 2)[-1] == t:
                found = idx
                break
            elif 'type' in v and t in v['properties']['type']['enum']:
                found = idx
                break

        if found is not None:
            for e in error.context:
                for el in reversed(e.absolute_schema_path):
                    if isinstance(el, int):
                        if el == found:
                            return e
                        break
    return error


Instruction:
Explain what the following Python 3 code does
def get_resource_manager(self, resource_type, data=None):
        if '.' in resource_type:
            provider_name, resource_type = resource_type.split('.', 1)
        else:
            provider_name = self.ctx.policy.provider_name

        provider_resources = clouds[provider_name].resources
        klass = provider_resources.get(resource_type)
        if klass is None:
            raise ValueError(resource_type)

        # if we're already querying via config carry it forward
        if not data and self.source_type == 'config' and getattr(
                klass.get_model(), 'config_type', None):
            return klass(self.ctx, {'source': self.source_type})
        return klass(self.ctx, data or {})

Response: get a resource manager or a given resource type


Instruction:
Can you create a Python 3 function that
augments ElasticBeanstalk Environments with their tags.

Response:
def _eb_env_tags(envs, session_factory, retry):
    """Augment ElasticBeanstalk Environments with their tags."""

    client = local_session(session_factory).client('elasticbeanstalk')

    def process_tags(eb_env):
        try:
            eb_env['Tags'] = retry(
                client.list_tags_for_resource,
                ResourceArn=eb_env['EnvironmentArn'])['ResourceTags']
        except client.exceptions.ResourceNotFoundException:
            return
        return eb_env

    # Handle API rate-limiting, which is a problem for accounts with many
    # EB Environments
    return list(map(process_tags, envs))


Instruction:
Write a Python 3 script to
assemble a document representing all the config state around a bucket.

Response:
def assemble_bucket(item):
    """Assemble a document representing all the config state around a bucket.

    TODO: Refactor this, the logic here feels quite muddled.
    """
    factory, b = item
    s = factory()
    c = s.client('s3')
    # Bucket Location, Current Client Location, Default Location
    b_location = c_location = location = "us-east-1"
    methods = list(S3_AUGMENT_TABLE)
    for m, k, default, select in methods:
        try:
            method = getattr(c, m)
            v = method(Bucket=b['Name'])
            v.pop('ResponseMetadata')
            if select is not None and select in v:
                v = v[select]
        except (ssl.SSLError, SSLError) as e:
            # Proxy issues? i assume
            log.warning("Bucket ssl error %s: %s %s",
                        b['Name'], b.get('Location', 'unknown'),
                        e)
            continue
        except ClientError as e:
            code = e.response['Error']['Code']
            if code.startswith("NoSuch") or "NotFound" in code:
                v = default
            elif code == 'PermanentRedirect':
                s = factory()
                c = bucket_client(s, b)
                # Requeue with the correct region given location constraint
                methods.append((m, k, default, select))
                continue
            else:
                log.warning(
                    "Bucket:%s unable to invoke method:%s error:%s ",
                    b['Name'], m, e.response['Error']['Message'])
                # For auth failures, we don't bail out, continue processing if we can.
                # Note this can lead to missing data, but in general is cleaner than
                # failing hard, due to the common use of locked down s3 bucket policies
                # that may cause issues fetching information across a fleet of buckets.

                # This does mean s3 policies depending on augments should check denied
                # methods annotation, generally though lacking get access to an augment means
                # they won't have write access either.

                # For other error types we raise and bail policy execution.
                if e.response['Error']['Code'] == 'AccessDenied':
                    b.setdefault('c7n:DeniedMethods', []).append(m)
                    continue
                raise
        # As soon as we learn location (which generally works)
        if k == 'Location' and v is not None:
            b_location = v.get('LocationConstraint')
            # Location == region for all cases but EU
            # https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGETlocation.html
            if b_location is None:
                b_location = "us-east-1"
            elif b_location == 'EU':
                b_location = "eu-west-1"
                v['LocationConstraint'] = 'eu-west-1'
            if v and v != c_location:
                c = s.client('s3', region_name=b_location)
            elif c_location != location:
                c = s.client('s3', region_name=location)
        b[k] = v
    return b


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_region(b):
    remap = {None: 'us-east-1', 'EU': 'eu-west-1'}
    region = b.get('Location', {}).get('LocationConstraint')
    return remap.get(region, region)

Response: Tries to get the region from Location. LocationConstraint EU defaults to us - east - 1


Instruction:
Can you generate a brief explanation for the following Python 3 code
def expand_permissions(self, permissions):
        for p in permissions:
            np = dict(p)
            values = {}
            for k in (u'IpRanges',
                      u'Ipv6Ranges',
                      u'PrefixListIds',
                      u'UserIdGroupPairs'):
                values[k] = np.pop(k, ())
                np[k] = []
            for k, v in values.items():
                if not v:
                    continue
                for e in v:
                    ep = dict(np)
                    ep[k] = [e]
                    yield ep

Response:
Expand each list of cidr prefix list user id group pair
        by port or protocol as an individual rule.


Instruction:
Can you tell what is the following Python 3 function doing
def report(policies, start_date, options, output_fh, raw_output_fh=None):
    regions = set([p.options.region for p in policies])
    policy_names = set([p.name for p in policies])
    formatter = Formatter(
        policies[0].resource_manager.resource_type,
        extra_fields=options.field,
        include_default_fields=not options.no_default_fields,
        include_region=len(regions) > 1,
        include_policy=len(policy_names) > 1
    )

    records = []
    for policy in policies:
        # initialize policy execution context for output access
        policy.ctx.initialize()
        if policy.ctx.output.type == 's3':
            policy_records = record_set(
                policy.session_factory,
                policy.ctx.output.config['netloc'],
                policy.ctx.output.config['path'].strip('/'),
                start_date)
        else:
            policy_records = fs_record_set(policy.ctx.log_dir, policy.name)

        log.debug("Found %d records for region %s", len(policy_records), policy.options.region)

        for record in policy_records:
            record['policy'] = policy.name
            record['region'] = policy.options.region

        records += policy_records

    rows = formatter.to_csv(records)

    if options.format == 'csv':
        writer = UnicodeWriter(output_fh, formatter.headers())
        writer.writerow(formatter.headers())
        writer.writerows(rows)
    elif options.format == 'json':
        print(dumps(records, indent=2))
    else:
        # We special case CSV, and for other formats we pass to tabulate
        print(tabulate(rows, formatter.headers(), tablefmt=options.format))

    if raw_output_fh is not None:
        dumps(records, raw_output_fh, indent=2)

Response: Format a list of policies into a single report.


Instruction:
Explain what the following Python 3 code does
def record_set(session_factory, bucket, key_prefix, start_date, specify_hour=False):

    s3 = local_session(session_factory).client('s3')

    records = []
    key_count = 0

    date = start_date.strftime('%Y/%m/%d')
    if specify_hour:
        date += "/{}".format(start_date.hour)
    else:
        date += "/00"

    marker = "{}/{}/resources.json.gz".format(key_prefix.strip("/"), date)

    p = s3.get_paginator('list_objects_v2').paginate(
        Bucket=bucket,
        Prefix=key_prefix.strip('/') + '/',
        StartAfter=marker,
    )

    with ThreadPoolExecutor(max_workers=20) as w:
        for key_set in p:
            if 'Contents' not in key_set:
                continue
            keys = [k for k in key_set['Contents']
                    if k['Key'].endswith('resources.json.gz')]
            key_count += len(keys)
            futures = map(lambda k: w.submit(
                get_records, bucket, k, session_factory), keys)

            for f in as_completed(futures):
                records.extend(f.result())

    log.info("Fetched %d records across %d files" % (
        len(records), key_count))
    return records

Response: Retrieve all s3 records for the given policy output url


Instruction:
Write a Python 3 script for
running org repo status hooks

Response:
def run(organization, hook_context, github_url, github_token,
        verbose, metrics=False, since=None, assume=None, region=None):
    """scan org repo status hooks"""
    logging.basicConfig(level=logging.DEBUG)

    since = dateparser.parse(
        since, settings={
            'RETURN_AS_TIMEZONE_AWARE': True, 'TO_TIMEZONE': 'UTC'})

    headers = {"Authorization": "token {}".format(github_token)}

    response = requests.post(
        github_url, headers=headers,
        json={'query': query, 'variables': {'organization': organization}})

    result = response.json()

    if response.status_code != 200 or 'errors' in result:
        raise Exception(
            "Query failed to run by returning code of {}. {}".format(
                response.status_code, response.content))

    now = datetime.utcnow().replace(tzinfo=tzutc())
    stats = Counter()
    repo_metrics = RepoMetrics(
        Bag(session_factory=SessionFactory(region, assume_role=assume)),
        {'namespace': DEFAULT_NAMESPACE}
    )

    for r in result['data']['organization']['repositories']['nodes']:
        commits = jmespath.search(
            'pullRequests.edges[].node[].commits[].nodes[].commit[]', r)
        if not commits:
            continue
        log.debug("processing repo: %s prs: %d", r['name'], len(commits))
        repo_metrics.dims = {
            'Hook': hook_context,
            'Repo': '{}/{}'.format(organization, r['name'])}

        # Each commit represents a separate pr
        for c in commits:
            process_commit(c, r, repo_metrics, stats, since, now)

    repo_metrics.dims = None

    if stats['missing']:
        repo_metrics.put_metric(
            'RepoHookPending', stats['missing'], 'Count',
            Hook=hook_context)
        repo_metrics.put_metric(
            'RepoHookLatency', stats['missing_time'], 'Seconds',
            Hook=hook_context)

    if not metrics:
        print(dumps(repo_metrics.buf, indent=2))
        return
    else:
        repo_metrics.BUF_SIZE = 20
        repo_metrics.flush()


Instruction:
Create a Python 3 function for
expanding any variables in the action to_from or cc_from fields.

Response:
def expand_variables(self, message):
        """expand any variables in the action to_from/cc_from fields.
        """
        p = copy.deepcopy(self.data)
        if 'to_from' in self.data:
            to_from = self.data['to_from'].copy()
            to_from['url'] = to_from['url'].format(**message)
            if 'expr' in to_from:
                to_from['expr'] = to_from['expr'].format(**message)
            p.setdefault('to', []).extend(ValuesFrom(to_from, self.manager).get_values())
        if 'cc_from' in self.data:
            cc_from = self.data['cc_from'].copy()
            cc_from['url'] = cc_from['url'].format(**message)
            if 'expr' in cc_from:
                cc_from['expr'] = cc_from['expr'].format(**message)
            p.setdefault('cc', []).extend(ValuesFrom(cc_from, self.manager).get_values())
        return p


Instruction:
Can you tell what is the following Python 3 function doing
def prepare_resources(self, resources):
        handler = getattr(self, "prepare_%s" % (
            self.manager.type.replace('-', '_')),
            None)
        if handler is None:
            return resources
        return handler(resources)

Response:
Prepare resources for transport.
       .


Instruction:
Can you generate the documentation for the following Python 3 function
def subscribe(config, accounts, region, merge, debug):
    config = validate.callback(config)
    subscription = config.get('subscription')

    if subscription is None:
        log.error("config file: logs subscription missing")
        sys.exit(1)

    def converge_destination_policy(client, config):
        destination_name = subscription['destination-arn'].rsplit(':', 1)[-1]
        try:
            extant_destinations = client.describe_destinations(
                DestinationNamePrefix=destination_name).get('destinations')
        except ClientError:
            log.error("Log group destination not found: %s",
                      subscription['destination-arn'])
            sys.exit(1)

        account_ids = set()
        for a in accounts:
            if isinstance(a['role'], list):
                account_ids.add(a['role'][-1].split(':')[4])
            else:
                account_ids.add(a['role'].split(':')[4])

        if merge:
            for d in extant_destinations:
                if d['destinationName'] == destination_name:
                    for s in json.loads(d['accessPolicy']):
                        if s['Sid'] == 'CrossAccountDelivery':
                            account_ids.update(s['Principal']['AWS'])

        client.put_destination_policy(
            destinationName=destination_name,
            accessPolicy=json.dumps({
                'Statement': [{
                    'Action': 'logs:PutSubscriptionFilter',
                    'Effect': 'Allow',
                    'Principal': {'AWS': list(account_ids)},
                    'Resource': subscription['destination-arn'],
                    'Sid': 'CrossAccountDelivery'}]}))

    def subscribe_account(t_account, subscription, region):
        session = get_session(t_account['role'], region)
        client = session.client('logs')
        distribution = subscription.get('distribution', 'ByLogStream')

        for g in account.get('groups'):
            if (g.endswith('*')):
                g = g.replace('*', '')
                paginator = client.get_paginator('describe_log_groups')
                allLogGroups = paginator.paginate(logGroupNamePrefix=g).build_full_result()
                for l in allLogGroups:
                    _process_subscribe_group(
                        client, l['logGroupName'], subscription, distribution)
            else:
                _process_subscribe_group(client, g, subscription, distribution)

    if subscription.get('managed-policy'):
        if subscription.get('destination-role'):
            session = get_session(subscription['destination-role'], region)
        else:
            session = boto3.Session()
        converge_destination_policy(session.client('logs'), config)

    executor = debug and MainThreadExecutor or ThreadPoolExecutor

    with executor(max_workers=32) as w:
        futures = {}
        for account in config.get('accounts', ()):
            if accounts and account['name'] not in accounts:
                continue
            futures[w.submit(subscribe_account, account, subscription, region)] = account

        for f in as_completed(futures):
            account = futures[f]
            if f.exception():
                log.error("Error on account %s err: %s",
                          account['name'], f.exception())
            log.info("Completed %s", account['name'])

Response: subscribe accounts log groups to target account log group destination


Instruction:
Can you generate a brief explanation for the following Python 3 code
def run(config, start, end, accounts, region, debug):
    config = validate.callback(config)
    destination = config.get('destination')
    start = start and parse(start) or start
    end = end and parse(end) or datetime.now()
    executor = debug and MainThreadExecutor or ThreadPoolExecutor
    with executor(max_workers=32) as w:
        futures = {}
        for account in config.get('accounts', ()):
            if accounts and account['name'] not in accounts:
                continue
            futures[
                w.submit(process_account, account, start,
                         end, destination, region)] = account
        for f in as_completed(futures):
            account = futures[f]
            if f.exception():
                log.error("Error on account %s err: %s",
                          account['name'], f.exception())
            log.info("Completed %s", account['name'])

Response: run export across accounts and log groups specified in config


Instruction:
Can you generate a brief explanation for the following Python 3 code
def lambdafan(func):
    if 'AWS_LAMBDA_FUNCTION_NAME' not in os.environ:
        return func

    @functools.wraps(func)
    def scaleout(*args, **kw):
        client = boto3.client('lambda')
        client.invoke(
            FunctionName=os.environ['AWS_LAMBDA_FUNCTION_NAME'],
            InvocationType='Event',
            Payload=dumps({
                'event': 'fanout',
                'function': func.__name__,
                'args': args,
                'kwargs': kw}),
            Qualifier=os.environ['AWS_LAMBDA_FUNCTION_VERSION'])
    return scaleout

Response: simple decorator that will auto fan out async style in lambda.


Instruction:
Explain what the following Python 3 code does
def filter_group_names(groups, patterns):
    group_names = [g['logGroupName'] for g in groups]
    matched = set()
    for p in patterns:
        matched.update(fnmatch.filter(group_names, p))
    return [g for g in groups if g['logGroupName'] in matched]

Response: Filter log groups by shell patterns.