Instruction:
Can you implement a function in Python 3 that
runs an aws script across accounts

Response:
def run_script(config, output_dir, accounts, tags, region, echo, serial, script_args):
    """run an aws script across accounts"""
    # TODO count up on success / error / error list by account
    accounts_config, custodian_config, executor = init(
        config, None, serial, True, accounts, tags, (), ())

    if echo:
        print("command to run: `%s`" % (" ".join(script_args)))
        return

    # Support fully quoted scripts, which are common to avoid parameter
    # overlap with c7n-org run-script.
    if len(script_args) == 1 and " " in script_args[0]:
        script_args = script_args[0].split()

    with executor(max_workers=WORKER_COUNT) as w:
        futures = {}
        for a in accounts_config.get('accounts', ()):
            for r in resolve_regions(region or a.get('regions', ())):
                futures[
                    w.submit(run_account_script, a, r, output_dir,
                             serial, script_args)] = (a, r)
        for f in as_completed(futures):
            a, r = futures[f]
            if f.exception():
                if serial:
                    raise
                log.warning(
                    "Error running script in %s @ %s exception: %s",
                    a['name'], r, f.exception())
            exit_code = f.result()
            if exit_code == 0:
                log.info(
                    "ran script on account:%s region:%s script: `%s`",
                    a['name'], r, " ".join(script_args))
            else:
                log.info(
                    "error running script on account:%s region:%s script: `%s`",
                    a['name'], r, " ".join(script_args))


Instruction:
How would you explain what the following Python 3 function does
def run_account(account, region, policies_config, output_path,
                cache_period, cache_path, metrics, dryrun, debug):
    """Execute a set of policies on an account.
    """
    logging.getLogger('custodian.output').setLevel(logging.ERROR + 1)
    CONN_CACHE.session = None
    CONN_CACHE.time = None

    # allow users to specify interpolated output paths
    if '{' not in output_path:
        output_path = os.path.join(output_path, account['name'], region)

    cache_path = os.path.join(cache_path, "%s-%s.cache" % (account['account_id'], region))

    config = Config.empty(
        region=region, cache=cache_path,
        cache_period=cache_period, dryrun=dryrun, output_dir=output_path,
        account_id=account['account_id'], metrics_enabled=metrics,
        log_group=None, profile=None, external_id=None)

    env_vars = account_tags(account)

    if account.get('role'):
        if isinstance(account['role'], six.string_types):
            config['assume_role'] = account['role']
            config['external_id'] = account.get('external_id')
        else:
            env_vars.update(
                _get_env_creds(get_session(account, 'custodian', region), region))

    elif account.get('profile'):
        config['profile'] = account['profile']

    policies = PolicyCollection.from_data(policies_config, config)
    policy_counts = {}
    success = True
    st = time.time()

    with environ(**env_vars):
        for p in policies:
            # Variable expansion and non schema validation (not optional)
            p.expand_variables(p.get_variables(account.get('vars', {})))
            p.validate()

            log.debug(
                "Running policy:%s account:%s region:%s",
                p.name, account['name'], region)
            try:
                resources = p.run()
                policy_counts[p.name] = resources and len(resources) or 0
                if not resources:
                    continue
                log.info(
                    "Ran account:%s region:%s policy:%s matched:%d time:%0.2f",
                    account['name'], region, p.name, len(resources),
                    time.time() - st)
            except ClientError as e:
                success = False
                if e.response['Error']['Code'] == 'AccessDenied':
                    log.warning('Access denied account:%s region:%s',
                                account['name'], region)
                    return policy_counts, success
                log.error(
                    "Exception running policy:%s account:%s region:%s error:%s",
                    p.name, account['name'], region, e)
                continue
            except Exception as e:
                success = False
                log.error(
                    "Exception running policy:%s account:%s region:%s error:%s",
                    p.name, account['name'], region, e)
                if not debug:
                    continue
                import traceback, pdb, sys
                traceback.print_exc()
                pdb.post_mortem(sys.exc_info()[-1])
                raise

    return policy_counts, success

Response: Execute a set of policies on an account.


Instruction:
Here you have a function in Python 3, explain what it does
def run(config, use, output_dir, accounts, tags, region,
        policy, policy_tags, cache_period, cache_path, metrics,
        dryrun, debug, verbose, metrics_uri):
    """run a custodian policy across accounts"""
    accounts_config, custodian_config, executor = init(
        config, use, debug, verbose, accounts, tags, policy, policy_tags=policy_tags)
    policy_counts = Counter()
    success = True

    if metrics_uri:
        metrics = metrics_uri

    if not cache_path:
        cache_path = os.path.expanduser("~/.cache/c7n-org")
        if not os.path.exists(cache_path):
            os.makedirs(cache_path)

    with executor(max_workers=WORKER_COUNT) as w:
        futures = {}
        for a in accounts_config['accounts']:
            for r in resolve_regions(region or a.get('regions', ())):
                futures[w.submit(
                    run_account,
                    a, r,
                    custodian_config,
                    output_dir,
                    cache_period,
                    cache_path,
                    metrics,
                    dryrun,
                    debug)] = (a, r)

        for f in as_completed(futures):
            a, r = futures[f]
            if f.exception():
                if debug:
                    raise
                log.warning(
                    "Error running policy in %s @ %s exception: %s",
                    a['name'], r, f.exception())

            account_region_pcounts, account_region_success = f.result()
            for p in account_region_pcounts:
                policy_counts[p] += account_region_pcounts[p]

            if not account_region_success:
                success = False

    log.info("Policy resource counts %s" % policy_counts)

    if not success:
        sys.exit(1)

Response: run a custodian policy across accounts


Instruction:
Can you generate a brief explanation for the following Python 3 code
def emit(self, message):
        # We're sending messages asynchronously, bubble to caller when
        # we've detected an error on the message. This isn't great,
        # but options once we've gone async without a deferred/promise
        # aren't great.
        if self.transport and self.transport.error:
            raise self.transport.error

        # Sanity safety, people do like to recurse by attaching to
        # root log :-(
        if message.name.startswith('boto'):
            return

        msg = self.format_message(message)
        if not self.transport:
            self.start_transports()
        self.buf.append(msg)
        self.flush_buffers(
            (message.created - self.last_seen >= self.batch_interval))

        self.last_seen = message.created

Response: Send a message to the log.


Instruction:
Create a Python 3 function to
ensure all logging output has been flushed.

Response:
def flush(self):
        """Ensure all logging output has been flushed."""
        if self.shutdown:
            return
        self.flush_buffers(force=True)
        self.queue.put(FLUSH_MARKER)
        self.queue.join()


Instruction:
Create a Python 3 function to
handle various client side errors when describing images

Response:
def extract_bad_ami(e):
        """Handle various client side errors when describing images"""
        msg = e.response['Error']['Message']
        error = e.response['Error']['Code']
        e_ami_ids = None
        if error == 'InvalidAMIID.NotFound':
            e_ami_ids = [
                e_ami_id.strip() for e_ami_id
                in msg[msg.find("'[") + 2:msg.rfind("]'")].split(',')]
            log.warning("Image not found %s" % e_ami_ids)
        elif error == 'InvalidAMIID.Malformed':
            e_ami_ids = [msg[msg.find('"') + 1:msg.rfind('"')]]
            log.warning("Image id malformed %s" % e_ami_ids)
        return e_ami_ids


Instruction:
Write a Python 3 script to
format config for lambda exec

Response:
def format_json(config):
    """format config for lambda exec
    """
    with open(config) as fh:
        print(json.dumps(yaml.safe_load(fh.read()), indent=2))


Instruction:
How would you explain what the following Python 3 function does
def metrics(function, api, start, period):
    from c7n.mu import LambdaManager
    manager = LambdaManager(boto3.Session)
    start = parse_date(start)
    period = int(abs(parse_timedelta(period).total_seconds()))

    print("Lambda Metrics")
    metrics = manager.metrics(
        [{'FunctionName': function}],
        start=start, end=datetime.utcnow(),
        period=period)
    for k in ('Invocations', 'Throttles', 'Errors'):
        values = [n['Sum'] for n in metrics[0][k]]
        render_metrics(k, values)

    if not api:
        return

    print("Api Metrics")
    metrics = gateway_metrics(
        boto3.Session, api, "latest", start, datetime.utcnow(), period)
    for k, data in metrics.items():
        if "Count" in k:
            values = [n['Sum'] for n in data]
        else:
            values = [n['Average'] for n in data]
        render_metrics(k, values)

    print("Db Metrics")
    metrics = db_metrics(
        boto3.Session, "Sphere11.Dev.ResourceLocks",
        start, datetime.utcnow(), period)
    for k, data in metrics.items():
        values = [n['Average'] for n in data]
        render_metrics(k, values)

Response: Get metrics for a given function in a given API.


Instruction:
Here you have a function in Python 3, explain what it does
def flush_pending(function):
    s = boto3.Session()
    client = s.client('lambda')
    results = client.invoke(
        FunctionName=function,
        Payload=json.dumps({'detail-type': 'Scheduled Event'})
    )
    content = results.pop('Payload').read()
    pprint.pprint(results)
    pprint.pprint(json.loads(content))

Response: Attempt to acquire any pending locks.


Instruction:
Make a summary of the following Python 3 code
def config_status():
    s = boto3.Session()
    client = s.client('config')
    channels = client.describe_delivery_channel_status()[
        'DeliveryChannelsStatus']
    for c in channels:
        print(yaml.safe_dump({
            c['name']: dict(
                snapshot=str(
                    c['configSnapshotDeliveryInfo'].get('lastSuccessfulTime')),
                history=str(
                    c['configHistoryDeliveryInfo'].get('lastSuccessfulTime')),
                stream=str(
                    c['configStreamDeliveryInfo'].get('lastStatusChangeTime'))
            ),
        }, default_flow_style=False))

Response: Check config status in an account.


Instruction:
Given the following Python 3 function, write the documentation
def local(reload, port):
    import logging
    from bottle import run
    from app import controller, app
    from c7n.resources import load_resources
    load_resources()
    print("Loaded resources definitions")
    logging.basicConfig(level=logging.DEBUG)
    logging.getLogger('botocore').setLevel(logging.WARNING)
    if controller.db.provision():
        print("Table Created")
    run(app, reloader=reload, port=port)

Response: run local app server


Instruction:
How would you explain what the following Python 3 function does
def _schema_get_docstring(starting_class):
    for cls in inspect.getmro(starting_class):
        if inspect.getdoc(cls):
            return inspect.getdoc(cls)

Response: Given a class return its docstring.


Instruction:
Can you implement a function in Python 3 that
returns a list of possible options for the given prefix.

Response:
def schema_completer(prefix):
    """ For tab-completion via argcomplete, return completion options.

    For the given prefix so far, return the possible options.  Note that
    filtering via startswith happens after this list is returned.
    """
    from c7n import schema
    load_resources()
    components = prefix.split('.')

    if components[0] in provider.clouds.keys():
        cloud_provider = components.pop(0)
        provider_resources = provider.resources(cloud_provider)
    else:
        cloud_provider = 'aws'
        provider_resources = provider.resources('aws')
        components[0] = "aws.%s" % components[0]

    # Completions for resource
    if len(components) == 1:
        choices = [r for r in provider.resources().keys()
                   if r.startswith(components[0])]
        if len(choices) == 1:
            choices += ['{}{}'.format(choices[0], '.')]
        return choices

    if components[0] not in provider_resources.keys():
        return []

    # Completions for category
    if len(components) == 2:
        choices = ['{}.{}'.format(components[0], x)
                   for x in ('actions', 'filters') if x.startswith(components[1])]
        if len(choices) == 1:
            choices += ['{}{}'.format(choices[0], '.')]
        return choices

    # Completions for item
    elif len(components) == 3:
        resource_mapping = schema.resource_vocabulary(cloud_provider)
        return ['{}.{}.{}'.format(components[0], components[1], x)
                for x in resource_mapping[components[0]][components[1]]]

    return []


Instruction:
Can you generate the documentation for the following Python 3 function
def schema_cmd(options):
    from c7n import schema
    if options.json:
        schema.json_dump(options.resource)
        return

    load_resources()

    resource_mapping = schema.resource_vocabulary()
    if options.summary:
        schema.summary(resource_mapping)
        return

    # Here are the formats for what we accept:
    # - No argument
    #   - List all available RESOURCES
    # - PROVIDER
    #   - List all available RESOURCES for supplied PROVIDER
    # - RESOURCE
    #   - List all available actions and filters for supplied RESOURCE
    # - MODE
    #   - List all available MODES
    # - RESOURCE.actions
    #   - List all available actions for supplied RESOURCE
    # - RESOURCE.actions.ACTION
    #   - Show class doc string and schema for supplied action
    # - RESOURCE.filters
    #   - List all available filters for supplied RESOURCE
    # - RESOURCE.filters.FILTER
    #   - Show class doc string and schema for supplied filter

    if not options.resource:
        resource_list = {'resources': sorted(provider.resources().keys())}
        print(yaml.safe_dump(resource_list, default_flow_style=False))
        return

    # Format is [PROVIDER].RESOURCE.CATEGORY.ITEM
    # optional provider defaults to aws for compatibility
    components = options.resource.lower().split('.')
    if len(components) == 1 and components[0] in provider.clouds.keys():
        resource_list = {'resources': sorted(
            provider.resources(cloud_provider=components[0]).keys())}
        print(yaml.safe_dump(resource_list, default_flow_style=False))
        return
    if components[0] in provider.clouds.keys():
        cloud_provider = components.pop(0)
        resource_mapping = schema.resource_vocabulary(
            cloud_provider)
        components[0] = '%s.%s' % (cloud_provider, components[0])
    elif components[0] in schema.resource_vocabulary().keys():
        resource_mapping = schema.resource_vocabulary()
    else:
        resource_mapping = schema.resource_vocabulary('aws')
        components[0] = 'aws.%s' % components[0]

    #
    # Handle mode
    #
    if components[0] == "mode":
        if len(components) == 1:
            output = {components[0]: list(resource_mapping[components[0]].keys())}
            print(yaml.safe_dump(output, default_flow_style=False))
            return

        if len(components) == 2:
            if components[1] not in resource_mapping[components[0]]:
                log.error('{} is not a valid mode'.format(components[1]))
                sys.exit(1)

            _print_cls_schema(resource_mapping[components[0]][components[1]])
            return

        # We received too much (e.g. mode.actions.foo)
        log.error("Invalid selector '{}'. Valid options are 'mode' "
                  "or 'mode.TYPE'".format(options.resource))
        sys.exit(1)
    #
    # Handle resource
    #
    resource = components[0]
    if resource not in resource_mapping:
        log.error('{} is not a valid resource'.format(resource))
        sys.exit(1)

    if len(components) == 1:
        docstring = _schema_get_docstring(
            resource_mapping[resource]['classes']['resource'])
        del(resource_mapping[resource]['classes'])
        if docstring:
            print("\nHelp\n----\n")
            print(docstring + '\n')
        output = {resource: resource_mapping[resource]}
        print(yaml.safe_dump(output))
        return

    #
    # Handle category
    #
    category = components[1]
    if category not in ('actions', 'filters'):
        log.error("Valid choices are 'actions' and 'filters'. You supplied '{}'".format(category))
        sys.exit(1)

    if len(components) == 2:
        output = "No {} available for resource {}.".format(category, resource)
        if category in resource_mapping[resource]:
            output = {resource: {
                category: resource_mapping[resource][category]}}
        print(yaml.safe_dump(output))
        return

    #
    # Handle item
    #
    item = components[2]
    if item not in resource_mapping[resource][category]:
        log.error('{} is not in the {} list for resource {}'.format(item, category, resource))
        sys.exit(1)

    if len(components) == 3:
        cls = resource_mapping[resource]['classes'][category][item]
        _print_cls_schema(cls)

        return

    # We received too much (e.g. s3.actions.foo.bar)
    log.error("Invalid selector '{}'.  Max of 3 components in the "
              "format RESOURCE.CATEGORY.ITEM".format(options.resource))
    sys.exit(1)

Response: Print info about the resources actions and filters available.


Instruction:
Implement a function in Python 3 to
determine the start and end dates based on user - supplied options.

Response:
def _metrics_get_endpoints(options):
    """ Determine the start and end dates based on user-supplied options. """
    if bool(options.start) ^ bool(options.end):
        log.error('--start and --end must be specified together')
        sys.exit(1)

    if options.start and options.end:
        start = options.start
        end = options.end
    else:
        end = datetime.utcnow()
        start = end - timedelta(options.days)

    return start, end


Instruction:
Can you create a Python 3 function that
extracts an instance id from an error

Response:
def extract_instance_id(state_error):
    "Extract an instance id from an error"
    instance_id = None
    match = RE_ERROR_INSTANCE_ID.search(str(state_error))
    if match:
        instance_id = match.groupdict().get('instance_id')
    if match is None or instance_id is None:
        raise ValueError("Could not extract instance id from error: %s" % state_error)
    return instance_id


Instruction:
Explain what the following Python 3 code does
def augment(self, resources):
        # First if we're in event based lambda go ahead and skip this,
        # tags can't be trusted in ec2 instances immediately post creation.
        if not resources or self.manager.data.get(
                'mode', {}).get('type', '') in (
                    'cloudtrail', 'ec2-instance-state'):
            return resources

        # AWOL detector, so we don't make extraneous api calls.
        resource_count = len(resources)
        search_count = min(int(resource_count % 0.05) + 1, 5)
        if search_count > resource_count:
            search_count = resource_count
        found = False
        for r in random.sample(resources, search_count):
            if 'Tags' in r:
                found = True
                break

        if found:
            return resources

        # Okay go and do the tag lookup
        client = utils.local_session(self.manager.session_factory).client('ec2')
        tag_set = self.manager.retry(
            client.describe_tags,
            Filters=[{'Name': 'resource-type',
                      'Values': ['instance']}])['Tags']
        resource_tags = {}
        for t in tag_set:
            t.pop('ResourceType')
            rid = t.pop('ResourceId')
            resource_tags.setdefault(rid, []).append(t)

        m = self.manager.get_model()
        for r in resources:
            r['Tags'] = resource_tags.get(r[m.id], ())
        return resources

Response: Augment the resource list with the tags from AWS EC2 API and AWOL API.


Instruction:
Here you have a function in Python 3, explain what it does
def register_resources(klass, registry, resource_class):
        for rtype, resource_manager in registry.items():
            if not resource_manager.has_arn():
                continue
            if 'post-finding' in resource_manager.action_registry:
                continue
            resource_class.filter_registry.register('finding', klass)

Response: register a new findings filter


Instruction:
Write a Python 3 function that can
publish a message to a GCP pub / sub topic

Response:
def publish_message(self, message, client):
        """Publish message to a GCP pub/sub topic
         """
        return client.execute_command('publish', {
            'topic': self.data['transport']['topic'],
            'body': {
                'messages': {
                    'data': self.pack(message)
                }
            }
        })


Instruction:
Create a Python 3 function for
setting headers on Mimetext message

Response:
def set_mimetext_headers(self, message, subject, from_addr, to_addrs, cc_addrs, priority):
        """Sets headers on Mimetext message"""

        message['Subject'] = subject
        message['From'] = from_addr
        message['To'] = ', '.join(to_addrs)
        if cc_addrs:
            message['Cc'] = ', '.join(cc_addrs)

        if priority and self.priority_header_is_valid(priority):
            priority = PRIORITIES[str(priority)].copy()
            for key in priority:
                message[key] = priority[key]

        return message


Instruction:
Explain what the following Python 3 code does
def register_resources(klass, registry, resource_class):
        services = {'acm-certificate', 'directconnect', 'dms-instance', 'directory', 'ec2',
                    'dynamodb-table', 'cache-cluster', 'efs', 'app-elb', 'elb', 'emr', 'rds',
                    'storage-gateway'}
        if resource_class.type in services:
            resource_class.filter_registry.register('health-event', klass)

Response: register_resources registers a health - event filter to the resources.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def custodian_archive(packages=None):
    modules = {'c7n', 'pkg_resources'}
    if packages:
        modules = filter(None, modules.union(packages))
    return PythonPackageArchive(*sorted(modules))

Response: Create a lambda code archive for running custodian.


Instruction:
Can you create a Python 3 function that
adds all. py files under the given directory path to the archive.

Response:
def add_directory(self, path, ignore=None):
        """Add ``*.py`` files under the directory ``path`` to the archive.
        """
        for root, dirs, files in os.walk(path):
            arc_prefix = os.path.relpath(root, os.path.dirname(path))
            # py3 remove pyc cache dirs.
            if '__pycache__' in dirs:
                dirs.remove('__pycache__')
            for f in files:
                dest_path = os.path.join(arc_prefix, f)

                # ignore specific files
                if ignore and ignore(dest_path):
                    continue

                if f.endswith('.pyc') or f.endswith('.c'):
                    continue
                f_path = os.path.join(root, f)

                self.add_file(f_path, dest_path)


Instruction:
Can you tell what is the following Python 3 function doing
def add_file(self, src, dest=None):
        dest = dest or os.path.basename(src)
        with open(src, 'rb') as fp:
            contents = fp.read()
        self.add_contents(dest, contents)

Response: Add the file at src to the archive.


Instruction:
How would you code a function in Python 3 to
add contents to the archive under dest.

Response:
def add_contents(self, dest, contents):
        """Add file contents to the archive under ``dest``.

        If ``dest`` is a path, it will be added compressed and world-readable
        (user-writeable). You may also pass a :py:class:`~zipfile.ZipInfo` for
        custom behavior.

        """
        assert not self._closed, "Archive closed"
        if not isinstance(dest, zipfile.ZipInfo):
            dest = zinfo(dest)  # see for some caveats
        # Ensure we apply the compression
        dest.compress_type = self.zip_compression
        # Mark host OS as Linux for all archives
        dest.create_system = 3
        self._zip_file.writestr(dest, contents)


Instruction:
Can you tell what is the following Python 3 function doing
def close(self):
        self._closed = True
        self._zip_file.close()
        log.debug(
            "Created custodian serverless archive size: %0.2fmb",
            (os.path.getsize(self._temp_archive_file.name) / (
                1024.0 * 1024.0)))
        return self

Response: Close the zip file and return the object.


Instruction:
Here you have a function in Python 3, explain what it does
def get_checksum(self, encoder=base64.b64encode, hasher=hashlib.sha256):
        assert self._closed, "Archive not closed"
        with open(self._temp_archive_file.name, 'rb') as fh:
            return encoder(checksum(fh, hasher())).decode('ascii')

Response: Return the b64 encoded sha256 checksum of the archive.


Instruction:
Write a Python 3 function that can
return a read - only : py : class : ~zipfile. ZipFile object.

Response:
def get_reader(self):
        """Return a read-only :py:class:`~zipfile.ZipFile`."""
        assert self._closed, "Archive not closed"
        buf = io.BytesIO(self.get_bytes())
        return zipfile.ZipFile(buf, mode='r')


Instruction:
Can you create a Python 3 function that
publishes an alias for the given function.

Response:
def publish_alias(self, func_data, alias):
        """Create or update an alias for the given function.
        """
        if not alias:
            return func_data['FunctionArn']
        func_name = func_data['FunctionName']
        func_version = func_data['Version']

        exists = resource_exists(
            self.client.get_alias, FunctionName=func_name, Name=alias)

        if not exists:
            log.debug("Publishing custodian lambda alias %s", alias)
            alias_result = self.client.create_alias(
                FunctionName=func_name,
                Name=alias,
                FunctionVersion=func_version)
        else:
            if (exists['FunctionVersion'] == func_version and
                    exists['Name'] == alias):
                return exists['AliasArn']
            log.debug('Updating custodian lambda alias %s', alias)
            alias_result = self.client.update_alias(
                FunctionName=func_name,
                Name=alias,
                FunctionVersion=func_version)
        return alias_result['AliasArn']


Instruction:
Make a summary of the following Python 3 code
def row_factory(cursor, row):
    d = {}
    for idx, col in enumerate(cursor.description):
        d[col[0]] = row[idx]
    return d

Response: Returns a sqlite row factory that returns a dictionary


Instruction:
How would you explain what the following Python 3 function does
def get_tag_value(resource, tag, utf_8=False):

        tags = {k.lower(): v for k, v in resource.get('tags', {}).items()}
        value = tags.get(tag, False)

        if value is not False:
            if utf_8:
                value = value.encode('utf8').decode('utf8')
        return value

Response: Get the resource s tag value.


Instruction:
Here you have a function in Python 3, explain what it does
def report(config, tags, accounts, master, debug, region):
    accounts_config, master_info, executor = guardian_init(
        config, debug, master, accounts, tags)

    session = get_session(
        master_info.get('role'), 'c7n-guardian',
        master_info.get('profile'),
        region)

    client = session.client('guardduty')
    detector_id = get_or_create_detector_id(client)

    members = {m['AccountId']: m for m in
               client.list_members(DetectorId=detector_id).get('Members')}

    accounts_report = []
    for a in accounts_config['accounts']:
        ar = dict(a)
        accounts_report.append(ar)
        ar.pop('tags', None)
        ar.pop('role')
        ar.pop('regions', None)
        if a['account_id'] not in members:
            ar['member'] = False
            ar['status'] = None
            ar['invited'] = None
            ar['updated'] = datetime.datetime.now().isoformat()
            continue
        m = members[a['account_id']]
        ar['status'] = m['RelationshipStatus']
        ar['member'] = True
        ar['joined'] = m['InvitedAt']
        ar['updated'] = m['UpdatedAt']

    accounts_report.sort(key=operator.itemgetter('updated'), reverse=True)
    print(tabulate(accounts_report, headers=('keys')))

Response: report on guard duty enablement by account


Instruction:
Can you generate the documentation for the following Python 3 function
def disable(config, tags, accounts, master, debug,
            suspend, disable_detector, delete_detector, dissociate, region):
    """suspend guard duty in the given accounts."""
    accounts_config, master_info, executor = guardian_init(
        config, debug, master, accounts, tags)

    if sum(map(int, (suspend, disable_detector, dissociate))) != 1:
        raise ValueError((
            "One and only of suspend, disable-detector, dissociate"
            "can be specified."))

    master_session = get_session(
        master_info['role'], 'c7n-guardian',
        master_info.get('profile'), region)
    master_client = master_session.client('guardduty')
    detector_id = get_or_create_detector_id(master_client)

    if suspend:
        unprocessed = master_client.stop_monitoring_members(
            DetectorId=detector_id,
            AccountIds=[a['account_id'] for a in accounts_config['accounts']]
        ).get('UnprocessedAccounts', ())

        if unprocessed:
            log.warning(
                "Following accounts where unprocessed\n %s",
                format_event(unprocessed))
        log.info("Stopped monitoring %d accounts in master",
                 len(accounts_config['accounts']))
        return

    if dissociate:
        master_client.disassociate_members(
            DetectorId=detector_id,
            AccountIds=[a['account_id'] for a in accounts_config['accounts']])

    # Seems like there's a couple of ways to disable an account
    # delete the detector (member), disable the detector (master or member),
    # or disassociate members, or from member disassociate from master.
    for a in accounts_config['accounts']:
        member_session = get_session(
            a['role'], 'c7n-guardian',
            a.get('profile'), region)

        member_client = member_session.client('guardduty')
        m_detector_id = get_or_create_detector_id(member_client)
        if disable_detector:
            member_client.update_detector(
                DetectorId=m_detector_id, Enable=False)
            log.info("Disabled detector in account:%s", a['name'])
        if dissociate:
            try:
                log.info("Disassociated member account:%s", a['name'])
                result = member_client.disassociate_from_master_account(
                    DetectorId=m_detector_id)
                log.info("Result %s", format_event(result))
            except ClientError as e:
                if e.response['Error']['Code'] == 'InvalidInputException':
                    continue
        if delete_detector:
            member_client.delete_detector(DetectorId=m_detector_id)
            log.info("Deleted detector in account:%s", a['name'])

Response: Disable guard duty in the given accounts.


Instruction:
Can you write a function in Python 3 where it
enables guard duty on a set of accounts

Response:
def enable(config, master, tags, accounts, debug, message, region):
    """enable guard duty on a set of accounts"""
    accounts_config, master_info, executor = guardian_init(
        config, debug, master, accounts, tags)
    regions = expand_regions(region)
    for r in regions:
        log.info("Processing Region:%s", r)
        enable_region(master_info, accounts_config, executor, message, r)


Instruction:
Implement a function in Python 3 to
return all the security group names configured in this action.

Response:
def get_action_group_names(self):
        """Return all the security group names configured in this action."""
        return self.get_group_names(
            list(itertools.chain(
                *[self._get_array('add'),
                  self._get_array('remove'),
                  self._get_array('isolation-group')])))


Instruction:
Make a summary of the following Python 3 code
def get_groups_by_names(self, names):
        if not names:
            return []
        client = utils.local_session(
            self.manager.session_factory).client('ec2')
        sgs = self.manager.retry(
            client.describe_security_groups,
            Filters=[{
                'Name': 'group-name', 'Values': names}]).get(
                    'SecurityGroups', [])

        unresolved = set(names)
        for s in sgs:
            if s['GroupName'] in unresolved:
                unresolved.remove(s['GroupName'])

        if unresolved:
            raise PolicyExecutionError(self._format_error(
                "policy:{policy} security groups not found "
                "requested: {names}, found: {groups}",
                names=list(unresolved), groups=[g['GroupId'] for g in sgs]))
        return sgs

Response: Resolve security names to security groups resources.


Instruction:
Given the following Python 3 function, write the documentation
def resolve_group_names(self, r, target_group_ids, groups):
        names = self.get_group_names(target_group_ids)
        if not names:
            return target_group_ids

        target_group_ids = list(target_group_ids)
        vpc_id = self.vpc_expr.search(r)
        if not vpc_id:
            raise PolicyExecutionError(self._format_error(
                "policy:{policy} non vpc attached resource used "
                "with modify-security-group: {resource_id}",
                resource_id=r[self.manager.resource_type.id]))

        found = False
        for n in names:
            for g in groups:
                if g['GroupName'] == n and g['VpcId'] == vpc_id:
                    found = g['GroupId']
            if not found:
                raise PolicyExecutionError(self._format_error((
                    "policy:{policy} could not resolve sg:{name} for "
                    "resource:{resource_id} in vpc:{vpc}"),
                    name=n,
                    resource_id=r[self.manager.resource_type.id], vpc=vpc_id))
            target_group_ids.remove(n)
            target_group_ids.append(found)
        return target_group_ids

Response:
Resolve any security group names to the corresponding group ids
            with the context of a given network attached resource.


Instruction:
Implement a Python 3 function for
resolving the resource security groups that need be modified.

Response:
def resolve_remove_symbols(self, r, target_group_ids, rgroups):
        """Resolve the resources security groups that need be modified.

        Specifically handles symbolic names that match annotations from policy filters
        for groups being removed.
        """
        if 'matched' in target_group_ids:
            return r.get('c7n:matched-security-groups', ())
        elif 'network-location' in target_group_ids:
            for reason in r.get('c7n:NetworkLocation', ()):
                if reason['reason'] == 'SecurityGroupMismatch':
                    return list(reason['security-groups'])
        elif 'all' in target_group_ids:
            return rgroups
        return target_group_ids


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_groups(self, resources):
        resolved_groups = self.get_groups_by_names(self.get_action_group_names())
        return_groups = []

        for idx, r in enumerate(resources):
            rgroups = self.sg_expr.search(r) or []
            add_groups = self.resolve_group_names(
                r, self._get_array('add'), resolved_groups)
            remove_groups = self.resolve_remove_symbols(
                r,
                self.resolve_group_names(
                    r, self._get_array('remove'), resolved_groups),
                rgroups)
            isolation_groups = self.resolve_group_names(
                r, self._get_array('isolation-group'), resolved_groups)

            for g in remove_groups:
                if g in rgroups:
                    rgroups.remove(g)
            for g in add_groups:
                if g not in rgroups:
                    rgroups.append(g)

            if not rgroups:
                rgroups = list(isolation_groups)

            return_groups.append(rgroups)

        return return_groups

Response: Returns a list of security groups that should be added to each resource.


Instruction:
Make a summary of the following Python 3 code
def type_schema(
        type_name, inherits=None, rinherit=None,
        aliases=None, required=None, **props):
    """jsonschema generation helper

    params:
     - type_name: name of the type
     - inherits: list of document fragments that are required via anyOf[$ref]
     - rinherit: use another schema as a base for this, basically work around
                 inherits issues with additionalProperties and type enums.
     - aliases: additional names this type maybe called
     - required: list of required properties, by default 'type' is required
     - props: additional key value properties
    """
    if aliases:
        type_names = [type_name]
        type_names.extend(aliases)
    else:
        type_names = [type_name]

    if rinherit:
        s = copy.deepcopy(rinherit)
        s['properties']['type'] = {'enum': type_names}
    else:
        s = {
            'type': 'object',
            'properties': {
                'type': {'enum': type_names}}}

    # Ref based inheritance and additional properties don't mix well.
    # https://stackoverflow.com/questions/22689900/json-schema-allof-with-additionalproperties
    if not inherits:
        s['additionalProperties'] = False

    s['properties'].update(props)
    if not required:
        required = []
    if isinstance(required, list):
        required.append('type')
    s['required'] = required
    if inherits:
        extended = s
        s = {'allOf': [{'$ref': i} for i in inherits]}
        s['allOf'].append(extended)
    return s

Response: generate a jsonschema for a type


Instruction:
Make a summary of the following Python 3 code
def group_by(resources, key):
    resource_map = {}
    parts = key.split('.')
    for r in resources:
        v = r
        for k in parts:
            v = v.get(k)
            if not isinstance(v, dict):
                break
        resource_map.setdefault(v, []).append(r)
    return resource_map

Response: Return a mapping of key value to resources with the corresponding value.


Instruction:
How would you implement a function in Python 3 that
queries EC2 instances for the query.

Response:
def query_instances(session, client=None, **query):
    """Return a list of ec2 instances for the query.
    """
    if client is None:
        client = session.client('ec2')
    p = client.get_paginator('describe_instances')
    results = p.paginate(**query)
    return list(itertools.chain(
        *[r["Instances"] for r in itertools.chain(
            *[pp['Reservations'] for pp in results])]))


Instruction:
How would you implement a function in Python 3 that
caches a session thread local for up to 45m

Response:
def local_session(factory):
    """Cache a session thread local for up to 45m"""
    factory_region = getattr(factory, 'region', 'global')
    s = getattr(CONN_CACHE, factory_region, {}).get('session')
    t = getattr(CONN_CACHE, factory_region, {}).get('time')

    n = time.time()
    if s is not None and t + (60 * 45) > n:
        return s
    s = factory()

    setattr(CONN_CACHE, factory_region, {'session': s, 'time': n})
    return s


Instruction:
Can you tell what is the following Python 3 function doing
def set_annotation(i, k, v):
    if not isinstance(i, dict):
        raise ValueError("Can only annotate dictionaries")

    if not isinstance(v, list):
        v = [v]

    if k in i:
        ev = i.get(k)
        if isinstance(ev, list):
            ev.extend(v)
    else:
        i[k] = v

Response: Set the annotation of a single object.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def generate_arn(
        service, resource, partition='aws',
        region=None, account_id=None, resource_type=None, separator='/'):
    """Generate an Amazon Resource Name.
    See http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html.
    """
    if region and region in REGION_PARTITION_MAP:
        partition = REGION_PARTITION_MAP[region]
    if service == 's3':
        region = ''
    arn = 'arn:%s:%s:%s:%s:' % (
        partition, service, region if region else '', account_id if account_id else '')
    if resource_type:
        arn = arn + '%s%s%s' % (resource_type, separator, resource)
    else:
        arn = arn + resource
    return arn

Response: Generate an Amazon Resource Name.


Instruction:
Implement a function in Python 3 to
return an identifier for a snapshot of a database or cluster.

Response:
def snapshot_identifier(prefix, db_identifier):
    """Return an identifier for a snapshot of a database or cluster.
    """
    now = datetime.now()
    return '%s-%s-%s' % (prefix, db_identifier, now.strftime('%Y-%m-%d-%H-%M'))


Instruction:
Can you implement a function in Python 3 that
reformats schema to be in a more displayable format.

Response:
def reformat_schema(model):
    """ Reformat schema to be in a more displayable format. """
    if not hasattr(model, 'schema'):
        return "Model '{}' does not have a schema".format(model)

    if 'properties' not in model.schema:
        return "Schema in unexpected format."

    ret = copy.deepcopy(model.schema['properties'])

    if 'type' in ret:
        del(ret['type'])

    for key in model.schema.get('required', []):
        if key in ret:
            ret[key]['required'] = True

    return ret


Instruction:
Implement a Python 3 function for
formatting all string values in an object.

Response:
def format_string_values(obj, err_fallback=(IndexError, KeyError), *args, **kwargs):
    """
    Format all string values in an object.
    Return the updated object
    """
    if isinstance(obj, dict):
        new = {}
        for key in obj.keys():
            new[key] = format_string_values(obj[key], *args, **kwargs)
        return new
    elif isinstance(obj, list):
        new = []
        for item in obj:
            new.append(format_string_values(item, *args, **kwargs))
        return new
    elif isinstance(obj, six.string_types):
        try:
            return obj.format(*args, **kwargs)
        except err_fallback:
            return obj
    else:
        return obj


Instruction:
Explain what the following Python 3 code does
def get_resources(self, ids, cache=True):
        client = local_session(self.manager.session_factory).client('dax')
        return client.describe_clusters(ClusterNames=ids).get('Clusters')

Response: Retrieve dax resources for serverless policies or related resources


Instruction:
Can you write a function in Python 3 where it
generates a c7n - org gcp projects config file

Response:
def main(output):
    """
    Generate a c7n-org gcp projects config file
    """

    client = Session().client('cloudresourcemanager', 'v1', 'projects')

    results = []
    for page in client.execute_paged_query('list', {}):
        for project in page.get('projects', []):

            if project['lifecycleState'] != 'ACTIVE':
                continue

            project_info = {
                'project_id': project['projectId'],
                'name': project['name'],
            }

            if 'labels' in project:
                project_info['tags'] = [
                    'label:%s:%s' % (k, v) for k, v in project.get('labels', {}).items()]
            results.append(project_info)

    output.write(
        yaml.safe_dump({'projects': results}, default_flow_style=False))


Instruction:
How would you code a function in Python 3 to
query a set of resources.

Response:
def filter(self, resource_manager, **params):
        """Query a set of resources."""
        m = self.resolve(resource_manager.resource_type)
        client = resource_manager.get_client()

        enum_op, list_op, extra_args = m.enum_spec

        parent_type, annotate_parent = m.parent_spec
        parents = self.manager.get_resource_manager(parent_type)

        # Have to query separately for each parent's children.
        results = []
        for parent in parents.resources():
            if extra_args:
                params.update({key: parent[extra_args[key]] for key in extra_args.keys()})

            op = getattr(getattr(client, enum_op), list_op)
            subset = [r.serialize(True) for r in op(**params)]

            if annotate_parent:
                for r in subset:
                    r[self.parent_key] = parent[parents.resource_type.id]

            if subset:
                results.extend(subset)
        return results


Instruction:
Given the following Python 3 function, write the documentation
def get_attr_filters(self):
        for f in self.data.keys():
            if f not in self.multi_attrs:
                continue
            fv = self.data[f]
            if isinstance(fv, dict):
                fv['key'] = f
            else:
                fv = {f: fv}
            vf = ValueFilter(fv)
            vf.annotate = False
            yield vf

Response: Return an iterator over resource attribute filters configured.


Instruction:
Create a Python 3 function to
add CloudTrail permissions to an S3 policy preserving existing

Response:
def cloudtrail_policy(original, bucket_name, account_id, bucket_region):
    '''add CloudTrail permissions to an S3 policy, preserving existing'''
    ct_actions = [
        {
            'Action': 's3:GetBucketAcl',
            'Effect': 'Allow',
            'Principal': {'Service': 'cloudtrail.amazonaws.com'},
            'Resource': generate_arn(
                service='s3', resource=bucket_name, region=bucket_region),
            'Sid': 'AWSCloudTrailAclCheck20150319',
        },
        {
            'Action': 's3:PutObject',
            'Condition': {
                'StringEquals':
                {'s3:x-amz-acl': 'bucket-owner-full-control'},
            },
            'Effect': 'Allow',
            'Principal': {'Service': 'cloudtrail.amazonaws.com'},
            'Resource': generate_arn(
                service='s3', resource=bucket_name, region=bucket_region),
            'Sid': 'AWSCloudTrailWrite20150319',
        },
    ]
    # parse original policy
    if original is None:
        policy = {
            'Statement': [],
            'Version': '2012-10-17',
        }
    else:
        policy = json.loads(original['Policy'])
    original_actions = [a.get('Action') for a in policy['Statement']]
    for cta in ct_actions:
        if cta['Action'] not in original_actions:
            policy['Statement'].append(cta)
    return json.dumps(policy)


Instruction:
How would you explain what the following Python 3 function does
def _get_available_engine_upgrades(client, major=False):
    results = {}
    engine_versions = client.describe_db_engine_versions()['DBEngineVersions']
    for v in engine_versions:
        if not v['Engine'] in results:
            results[v['Engine']] = {}
        if 'ValidUpgradeTarget' not in v or len(v['ValidUpgradeTarget']) == 0:
            continue
        for t in v['ValidUpgradeTarget']:
            if not major and t['IsMajorVersionUpgrade']:
                continue
            if LooseVersion(t['EngineVersion']) > LooseVersion(
                    results[v['Engine']].get(v['EngineVersion'], '0.0.0')):
                results[v['Engine']][v['EngineVersion']] = t['EngineVersion']
    return results

Response: Returns all available rds engine upgrades.


Instruction:
Can you generate the documentation for the following Python 3 function
def get_local_output_dir():
    output_dir = os.environ.get('C7N_OUTPUT_DIR', '/tmp/' + str(uuid.uuid4()))
    if not os.path.exists(output_dir):
        try:
            os.mkdir(output_dir)
        except OSError as error:
            log.warning("Unable to make output directory: {}".format(error))
    return output_dir

Response: Create a local output directory per execution.


Instruction:
Create a Python 3 function to
initialize the policy lambda execution configuration.

Response:
def init_config(policy_config):
    """Get policy lambda execution configuration.

    cli parameters are serialized into the policy lambda config,
    we merge those with any policy specific execution options.

    --assume role and -s output directory get special handling, as
    to disambiguate any cli context.

    account id is sourced from the config options or from api call
    and cached as a global
    """
    global account_id

    exec_options = policy_config.get('execution-options', {})

    # Remove some configuration options that don't make sense to translate from
    # cli to lambda automatically.
    #  - assume role on cli doesn't translate, it is the default lambda role and
    #    used to provision the lambda.
    #  - profile doesnt translate to lambda its `home` dir setup dependent
    #  - dryrun doesn't translate (and shouldn't be present)
    #  - region doesn't translate from cli (the lambda is bound to a region), and
    #    on the cli represents the region the lambda is provisioned in.
    for k in ('assume_role', 'profile', 'region', 'dryrun', 'cache'):
        exec_options.pop(k, None)

    # a cli local directory doesn't translate to lambda
    if not exec_options.get('output_dir', '').startswith('s3'):
        exec_options['output_dir'] = get_local_output_dir()

    # we can source account id from the cli parameters to avoid the sts call
    if exec_options.get('account_id'):
        account_id = exec_options['account_id']

    # merge with policy specific configuration
    exec_options.update(
        policy_config['policies'][0].get('mode', {}).get('execution-options', {}))

    # if using assume role in lambda ensure that the correct
    # execution account is captured in options.
    if 'assume_role' in exec_options:
        account_id = exec_options['assume_role'].split(':')[4]
    elif account_id is None:
        session = boto3.Session()
        account_id = get_account_id_from_sts(session)
    exec_options['account_id'] = account_id

    # Historical compatibility with manually set execution options
    # previously this was a boolean, its now a string value with the
    # boolean flag triggering a string value of 'aws'
    if 'metrics_enabled' in exec_options \
       and isinstance(exec_options['metrics_enabled'], bool) \
       and exec_options['metrics_enabled']:
        exec_options['metrics_enabled'] = 'aws'

    return Config.empty(**exec_options)


Instruction:
Write a Python 3 script for
indexing policy metrics in a given time range

Response:
def index_metrics(
        config, start, end, incremental=False, concurrency=5, accounts=None,
        period=3600, tag=None, index='policy-metrics', verbose=False):
    """index policy metrics"""
    logging.basicConfig(level=(verbose and logging.DEBUG or logging.INFO))
    logging.getLogger('botocore').setLevel(logging.WARNING)
    logging.getLogger('elasticsearch').setLevel(logging.WARNING)
    logging.getLogger('urllib3').setLevel(logging.WARNING)
    logging.getLogger('requests').setLevel(logging.WARNING)
    logging.getLogger('c7n.worker').setLevel(logging.INFO)

    with open(config) as fh:
        config = yaml.safe_load(fh.read())
    jsonschema.validate(config, CONFIG_SCHEMA)

    start, end = get_date_range(start, end)

    p_accounts = set()
    p_account_stats = {}
    i_time = i_points = 0
    t = time.time()

    with ProcessPoolExecutor(max_workers=concurrency) as w:
        futures = {}
        jobs = []
        # Filter
        for account in config.get('accounts'):
            if accounts and account['name'] not in accounts:
                continue
            if tag:
                found = False
                for t in account['tags'].values():
                    if tag == t:
                        found = True
                        break
                if not found:
                    continue
            p_accounts.add((account['name']))
            for region in account.get('regions'):
                for (p_start, p_end) in get_periods(start, end, period):
                    p = (config, index, region, account, p_start, p_end, period)
                    jobs.append(p)

        # by default we'll be effectively processing in order, but thats bumps
        # our concurrency into rate limits on metrics retrieval in a given account
        # region, go ahead and shuffle, at least with lucene, the non ordering
        # should have minimal impact on query perf (inverted index).

        random.shuffle(jobs)

        for j in jobs:
            log.debug("submit account:%s region:%s start:%s end:%s" % (
                j[3]['name'], j[2], j[4], j[5]))
            futures[w.submit(index_account_metrics, *j)] = j

        # Process completed
        for f in as_completed(futures):
            config, index, region, account, p_start, p_end, period = futures[f]
            if f.exception():
                log.warning("error account:%s region:%s error:%s",
                            account['name'], region, f.exception())
                continue
            rtime, rpoints = f.result()
            rstat = p_account_stats.setdefault(
                account['name'], {}).setdefault(region, {'points': 0})
            rstat['points'] += rpoints

            # log.info("complete account:%s, region:%s points:%s time:%0.2f",
            #         account['name'], region, rpoints, rtime)

            i_time += rtime
            i_points += rpoints

        log.info("complete accounts:%d points:%d itime:%0.2f time:%0.2f",
                 len(p_accounts), i_points, i_time, time.time() - t)


Instruction:
Create a Python 3 function for
making config revision look like describe output.

Response:
def transform_revision(self, revision):
        """make config revision look like describe output."""
        config = self.manager.get_source('config')
        return config.load_resource(revision)


Instruction:
Make a summary of the following Python 3 code
def register_resources(klass, registry, resource_class):
        config_type = getattr(resource_class.resource_type, 'config_type', None)
        if config_type is None:
            return
        resource_class.filter_registry.register('json-diff', klass)

Response: register the json - diff filter


Instruction:
Can you generate the documentation for the following Python 3 function
def _default_options(p, blacklist=""):
    provider = p.add_argument_group(
        "provider", "AWS account information, defaults per the aws cli")

    if 'region' not in blacklist:
        provider.add_argument(
            "-r", "--region", action='append', default=[],
            dest='regions', metavar='REGION',
            help="AWS Region to target.  Can be used multiple times")
    provider.add_argument(
        "--profile",
        help="AWS Account Config File Profile to utilize")
    provider.add_argument("--assume", default=None, dest="assume_role",
                          help="Role to assume")
    provider.add_argument("--external-id", default=None, dest="external_id",
                          help="External Id to provide when assuming a role")

    config = p.add_argument_group(
        "config", "Policy config file(s) and policy selectors")
    # -c is deprecated.  Supported for legacy reasons
    config.add_argument("-c", "--config", help=argparse.SUPPRESS)
    config.add_argument("configs", nargs='*',
                        help="Policy configuration file(s)")
    config.add_argument("-p", "--policies", default=None, dest='policy_filter',
                        help="Only use named/matched policies")
    config.add_argument("-t", "--resource", default=None, dest='resource_type',
                        help="Only use policies with the given resource type")

    output = p.add_argument_group("output", "Output control")
    output.add_argument("-v", "--verbose", action="count", help="Verbose logging")
    if 'quiet' not in blacklist:
        output.add_argument("-q", "--quiet", action="count",
                            help="Less logging (repeatable, -qqq for no output)")
    else:
        output.add_argument("-q", "--quiet", action="count", help=argparse.SUPPRESS)
    output.add_argument("--debug", default=False, help=argparse.SUPPRESS,
                        action="store_true")

    if 'vars' not in blacklist:
        # p.add_argument('--vars', default=None,
        #               help='Vars file to substitute into policy')
        p.set_defaults(vars=None)

    if 'log-group' not in blacklist:
        p.add_argument(
            "-l", "--log-group", default=None,
            help="Cloudwatch Log Group to send policy logs")
    else:
        p.add_argument("--log-group", default=None, help=argparse.SUPPRESS)

    if 'output-dir' not in blacklist:
        p.add_argument("-s", "--output-dir", required=True,
                       help="[REQUIRED] Directory or S3 URL For policy output")

    if 'cache' not in blacklist:
        p.add_argument(
            "-f", "--cache", default="~/.cache/cloud-custodian.cache",
            help="Cache file (default %(default)s)")
        p.add_argument(
            "--cache-period", default=15, type=int,
            help="Cache validity in minutes (default %(default)i)")
    else:
        p.add_argument("--cache", default=None, help=argparse.SUPPRESS)

Response: Add basic options ot the subparser.


Instruction:
Can you create a Python 3 function that
adds options specific to the report subcommand.

Response:
def _report_options(p):
    """ Add options specific to the report subcommand. """
    _default_options(p, blacklist=['cache', 'log-group', 'quiet'])
    p.add_argument(
        '--days', type=float, default=1,
        help="Number of days of history to consider")
    p.add_argument(
        '--raw', type=argparse.FileType('wb'),
        help="Store raw json of collected records to given file path")
    p.add_argument(
        '--field', action='append', default=[], type=_key_val_pair,
        metavar='HEADER=FIELD',
        help='Repeatable. JMESPath of field to include in the output OR '
        'for a tag use prefix `tag:`. Special case fields `region` and'
        '`policy` are available')
    p.add_argument(
        '--no-default-fields', action="store_true",
        help='Exclude default fields for report.')
    p.add_argument(
        '--format', default='csv', choices=['csv', 'grid', 'simple', 'json'],
        help="Format to output data in (default: %(default)s). "
        "Options include simple, grid, csv, json")


Instruction:
Can you implement a function in Python 3 that
adds options specific to metrics subcommand.

Response:
def _metrics_options(p):
    """ Add options specific to metrics subcommand. """
    _default_options(p, blacklist=['log-group', 'output-dir', 'cache', 'quiet'])

    p.add_argument(
        '--start', type=date_parse,
        help='Start date (requires --end, overrides --days)')
    p.add_argument(
        '--end', type=date_parse, help='End date')
    p.add_argument(
        '--days', type=int, default=14,
        help='Number of days of history to consider (default: %(default)i)')
    p.add_argument('--period', type=int, default=60 * 24 * 24)


Instruction:
Can you write a function in Python 3 where it
adds options specific to logs subcommand.

Response:
def _logs_options(p):
    """ Add options specific to logs subcommand. """
    _default_options(p, blacklist=['cache', 'quiet'])

    # default time range is 0 to "now" (to include all log entries)
    p.add_argument(
        '--start',
        default='the beginning',  # invalid, will result in 0
        help='Start date and/or time',
    )
    p.add_argument(
        '--end',
        default=datetime.now().strftime('%c'),
        help='End date and/or time',
    )


Instruction:
Write a Python 3 script to
add options specific to schema subcommand.

Response:
def _schema_options(p):
    """ Add options specific to schema subcommand. """

    p.add_argument(
        'resource', metavar='selector', nargs='?',
        default=None).completer = _schema_tab_completer
    p.add_argument(
        '--summary', action="store_true",
        help="Summarize counts of available resources, actions and filters")
    p.add_argument('--json', action="store_true", help=argparse.SUPPRESS)
    p.add_argument("-v", "--verbose", action="count", help="Verbose logging")
    p.add_argument("-q", "--quiet", action="count", help=argparse.SUPPRESS)
    p.add_argument("--debug", default=False, help=argparse.SUPPRESS)


Instruction:
How would you implement a function in Python 3 that
augments rds clusters with their respective tags.

Response:
def _rds_cluster_tags(model, dbs, session_factory, generator, retry):
    """Augment rds clusters with their respective tags."""
    client = local_session(session_factory).client('rds')

    def process_tags(db):
        try:
            db['Tags'] = retry(
                client.list_tags_for_resource,
                ResourceName=generator(db[model.id]))['TagList']
            return db
        except client.exceptions.DBClusterNotFoundFault:
            return None

    # Rds maintains a low api call limit, so this can take some time :-(
    return list(filter(None, map(process_tags, dbs)))


Instruction:
Explain what the following Python 3 code does
def process_user_record(cls, info):
        keys = list(info.keys())
        # Value conversion
        for k in keys:
            v = info[k]
            if v in ('N/A', 'no_information'):
                info[k] = None
            elif v == 'false':
                info[k] = False
            elif v == 'true':
                info[k] = True
        # Object conversion
        for p, t in cls.list_sub_objects:
            obj = dict([(k[len(p):], info.pop(k))
                        for k in keys if k.startswith(p)])
            if obj.get('active', False):
                info.setdefault(t, []).append(obj)
        return info

Response: Type convert the csv record modifies in place.


Instruction:
How would you explain what the following Python 3 function does
def normalized_log_entries(raw_entries):
    '''Mimic the format returned by LambdaManager.logs()'''
    entry_start = r'([0-9:, \-]+) - .* - (\w+) - (.*)$'
    entry = None
    # process start/end here - avoid parsing log entries twice
    for line in raw_entries:
        m = re.match(entry_start, line)
        if m:
            # this is the start of a new entry
            # spit out the one previously built up (if any)
            if entry is not None:
                yield entry
            (log_time, log_level, log_text) = m.groups()
            # convert time
            log_timestamp = _timestamp_from_string(log_time)
            # join level and first line of message
            msg = '[{}] {}'.format(log_level, log_text)
            entry = {
                'timestamp': log_timestamp,
                'message': msg,
            }
        else:
            # additional line(s) for entry (i.e. stack trace)
            entry['message'] = entry['message'] + line
    if entry is not None:
        # return the final entry
        yield entry

Response: Mimic the format returned by LambdaManager. logs


Instruction:
How would you explain what the following Python 3 function does
def log_entries_in_range(entries, start, end):
    '''filter out entries before start and after end'''
    start = _timestamp_from_string(start)
    end = _timestamp_from_string(end)
    for entry in entries:
        log_timestamp = entry.get('timestamp', 0)
        if log_timestamp >= start and log_timestamp <= end:
            yield entry

Response: filter out entries in a given range


Instruction:
Create a Python 3 function to
get log entries from a specific log group.

Response:
def log_entries_from_group(session, group_name, start, end):
    '''Get logs for a specific log group'''
    logs = session.client('logs')
    log.info("Fetching logs from group: %s" % group_name)
    try:
        logs.describe_log_groups(logGroupNamePrefix=group_name)
    except ClientError as e:
        if e.response['Error']['Code'] == 'ResourceNotFoundException':
            return
        raise
    try:
        log_streams = logs.describe_log_streams(
            logGroupName=group_name,
            orderBy="LastEventTime",
            limit=3,
            descending=True,
        )
    except ClientError as e:
        if e.response['Error']['Code'] == 'ResourceNotFoundException':
            return
        raise
    start = _timestamp_from_string(start)
    end = _timestamp_from_string(end)
    for s in reversed(log_streams['logStreams']):
        result = logs.get_log_events(
            logGroupName=group_name,
            logStreamName=s['logStreamName'],
            startTime=start,
            endTime=end,
        )
        for e in result['events']:
            yield e


Instruction:
Write a Python 3 script to
build and returns a cloud API service object.

Response:
def _create_service_api(credentials, service_name, version, developer_key=None,
                        cache_discovery=False, http=None):
    """Builds and returns a cloud API service object.

    Args:
        credentials (OAuth2Credentials): Credentials that will be used to
            authenticate the API calls.
        service_name (str): The name of the API.
        version (str): The version of the API to use.
        developer_key (str): The api key to use to determine the project
            associated with the API call, most API services do not require
            this to be set.
        cache_discovery (bool): Whether or not to cache the discovery doc.

    Returns:
        object: A Resource object with methods for interacting with the service.
    """
    # The default logging of the discovery obj is very noisy in recent versions.
    # Lower the default logging level of just this module to WARNING unless
    # debug is enabled.
    if log.getEffectiveLevel() > logging.DEBUG:
        logging.getLogger(discovery.__name__).setLevel(logging.WARNING)

    discovery_kwargs = {
        'serviceName': service_name,
        'version': version,
        'developerKey': developer_key,
        'cache_discovery': cache_discovery,
    }

    if http:
        discovery_kwargs['http'] = http
    else:
        discovery_kwargs['credentials'] = credentials

    return discovery.build(**discovery_kwargs)


Instruction:
Can you write a function in Python 3 where it
constructs an http client suitable for googleapiclient usage w / user agent.

Response:
def _build_http(http=None):
    """Construct an http client suitable for googleapiclient usage w/ user agent.
    """
    if not http:
        http = httplib2.Http(
            timeout=HTTP_REQUEST_TIMEOUT, ca_certs=HTTPLIB_CA_BUNDLE)

    user_agent = 'Python-httplib2/{} (gzip), {}/{}'.format(
        httplib2.__version__,
        'custodian-gcp',
        '0.1')
    return set_user_agent(http, user_agent)


Instruction:
Can you tell what is the following Python 3 function doing
def http(self):
        if self._use_cached_http and hasattr(self._local, 'http'):
            return self._local.http
        if self._http_replay is not None:
            # httplib2 instance is not thread safe
            http = self._http_replay
        else:
            http = _build_http()
        authorized_http = google_auth_httplib2.AuthorizedHttp(
            self._credentials, http=http)
        if self._use_cached_http:
            self._local.http = authorized_http
        return authorized_http

Response: A httplib2. Http instance authorized by the credentials.


Instruction:
How would you explain what the following Python 3 function does
def _build_request(self, verb, verb_arguments):
        method = getattr(self._component, verb)

        # Python insists that keys in **kwargs be strings (not variables).
        # Since we initially build our kwargs as a dictionary where one of the
        # keys is a variable (target), we need to convert keys to strings,
        # even though the variable in question is of type str.
        method_args = {str(k): v for k, v in verb_arguments.items()}
        return method(**method_args)

Response: Builds a HttpRequest object.


Instruction:
Create a Python 3 function for
building the next request object.

Response:
def _build_next_request(self, verb, prior_request, prior_response):
        """Builds pagination-aware request object.

        More details:
          https://developers.google.com/api-client-library/python/guide/pagination

        Args:
            verb (str): Request verb (ex. insert, update, delete).
            prior_request (httplib2.HttpRequest): Request that may trigger
                paging.
            prior_response (dict): Potentially partial response.

        Returns:
            httplib2.HttpRequest: HttpRequest or None. None is returned when
                there is nothing more to fetch - request completed.
        """
        method = getattr(self._component, verb + '_next')
        return method(prior_request, prior_response)


Instruction:
Can you create a Python 3 function that
executes a command via a dedicated http object.

Response:
def execute_command(self, verb, verb_arguments):
        """Executes command (ex. add) via a dedicated http object.

        Async APIs may take minutes to complete. Therefore, callers are
        encouraged to leverage concurrent.futures (or similar) to place long
        running commands on a separate threads.

        Args:
            verb (str): Method to execute on the component (ex. get, list).
            verb_arguments (dict): key-value pairs to be passed to _build_request.

        Returns:
            dict: An async operation Service Response.
        """
        request = self._build_request(verb, verb_arguments)
        return self._execute(request)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def execute_paged_query(self, verb, verb_arguments):
        if not self.supports_pagination(verb=verb):
            raise PaginationNotSupported('{} does not support pagination')

        request = self._build_request(verb, verb_arguments)

        number_of_pages_processed = 0
        while request is not None:
            response = self._execute(request)
            number_of_pages_processed += 1
            log.debug('Executing paged request #%s', number_of_pages_processed)
            request = self._build_next_request(verb, request, response)
            yield response

Response: Executes a paginated query via a dedicated http object.


Instruction:
Can you write a function in Python 3 where it
executes a query via a dedicated http object.

Response:
def execute_search_query(self, verb, verb_arguments):
        """Executes query (ex. search) via a dedicated http object.

        Args:
            verb (str): Method to execute on the component (ex. search).
            verb_arguments (dict): key-value pairs to be passed to _BuildRequest.

        Yields:
            dict: Service Response.
        """
        # Implementation of search does not follow the standard API pattern.
        # Fields need to be in the body rather than sent seperately.
        next_page_token = None
        number_of_pages_processed = 0
        while True:
            req_body = verb_arguments.get('body', dict())
            if next_page_token:
                req_body['pageToken'] = next_page_token
            request = self._build_request(verb, verb_arguments)
            response = self._execute(request)
            number_of_pages_processed += 1
            log.debug('Executing paged request #%s', number_of_pages_processed)
            next_page_token = response.get('nextPageToken')
            yield response

            if not next_page_token:
                break


Instruction:
Here you have a function in Python 3, explain what it does
def execute_query(self, verb, verb_arguments):
        request = self._build_request(verb, verb_arguments)
        return self._execute(request)

Response: Executes a query via a dedicated http object.


Instruction:
Implement a Python 3 function for
executing with retries and rate limiting.

Response:
def _execute(self, request):
        """Run execute with retries and rate limiting.

        Args:
            request (object): The HttpRequest object to execute.

        Returns:
            dict: The response from the API.
        """
        if self._rate_limiter:
            # Since the ratelimiter library only exposes a context manager
            # interface the code has to be duplicated to handle the case where
            # no rate limiter is defined.
            with self._rate_limiter:
                return request.execute(http=self.http,
                                       num_retries=self._num_retries)
        return request.execute(http=self.http,
                               num_retries=self._num_retries)


Instruction:
Can you generate the documentation for the following Python 3 function
def info(self, account_id, resource_id, parent_id):
        resource = self.record(account_id, resource_id)
        if resource is None and not parent_id:
            return {'ResourceId': resource_id,
                    'LockStatus': self.STATE_UNLOCKED}
        elif resource is None:
            parent = self.record(account_id, parent_id)
            if parent is None:
                return {'ResourceId': resource_id,
                        'ParentId': parent_id,
                        'LockStatus': self.STATE_UNLOCKED}
            parent['ResourceId'] = resource_id
            parent['ParentId'] = parent_id
            parent['LockType'] = 'parent'
            return parent
        if resource['ResourceId'].startswith('vpc-'):
            return resource
        if resource['ResourceId'].startswith('sg-'):
            return resource

Response: Get the resource lock status.


Instruction:
Explain what the following Python 3 code does
def process_eni_metrics(
        stream_eni, myips, stream,
        start, end, period, sample_size,
        resolver, sink_uri):
    """ENI flow stream processor that rollups, enhances,
       and indexes the stream by time period."""
    stats = Counter()
    period_counters = flow_stream_stats(myips, stream, period)
    client = InfluxDBClient.from_dsn(sink_uri)
    resource = resolver.resolve_resource(stream_eni)
    points = []

    for period in sorted(period_counters):
        pc = period_counters[period]
        pd = datetime.fromtimestamp(period)

        for t in ('inbytes', 'outbytes'):
            tpc = pc[t]
            ips = [ip for ip, _ in tpc.most_common(sample_size)]
            resolved = resolver.resolve(ips, pd - timedelta(900), pd + timedelta(900))
            logical_counter = rollup_logical(tpc, resolved, ('app', 'env'))
            for (app, env), v in logical_counter.items():
                p = {}
#                rinfo = resolved.get(ip, {})
                p['fields'] = {'Bytes': v}
                p['measurement'] = 'traffic_%s' % t
                p['time'] = datetime.fromtimestamp(period)
                p['tags'] = {
                    'Kind': resource['type'],
                    'AccountId': resource['account_id'],
                    'App': resource['app'],
                    'Env': resource['env'],
                    'ForeignApp': app,
                    'ForeignEnv': env}
                points.append(p)

        if len(points) > 2000:
            client.write_points(points)
            stats['Points'] += len(points)
            points = []

    client.write_points(points)
    stats['Points'] += len(points)
    log.info('periods:%d resource:%s points:%d',
             len(period_counters), resource, stats['Points'])
    return stats

Response: ENI flow stream processor that rollups enhances and indexes the stream by time period.


Instruction:
Explain what the following Python 3 code does
def analyze_app(
        app, env, account_id,
        bucket, prefix, store_dir,
        resources, ipdb, ipranges,
        start, end, tz,
        sink, period, sample_count,
        debug):
    """Analyze flow log records for application and generate metrics per period"""
    logging.basicConfig(level=logging.INFO)
    logging.getLogger('botocore').setLevel(logging.WARNING)

    executor = debug and MainThreadExecutor or ThreadPoolExecutor
    start, end = get_dates(start, end, tz)
    resolver = IPResolver(ipdb, ipdb, ipranges)

    for rtype_name in resources:
        rtype = Resource.get_type(rtype_name)
        resource_map = {
            rtype.id(r): r for r
            in rtype.get_resources(ipdb, start, end, app, env)}
        log.info("App:%s Env:%s Type:%s Found:%d",
                 app, env, rtype_name, len(resource_map))

        with sqlite3.connect(ipdb) as db:
            db.row_factory = row_factory
            cursor = db.cursor()
            cursor.execute(
                'select * from enis where resource_type in (%s)' % (
                    ", ".join(["'%s'" % r for r in resource_map.keys()])))
            enis = list(cursor)
            eni_map = {e['eni_id']: e for e in enis}

        # TODO: Download should be doing date bits here across the range of days.
        log_prefix = "%s/%s/flow-log/%s/%s" % (
            prefix.rstrip('/'),
            account_id,
            start.strftime('%Y/%m/%d'),
            "00000000-0000-0000-0000-000000000000")

        f_downloads = {}
        f_metrics = {}
        files = {}

        # should probably just queue this out to distributed worker pool
        with executor(max_workers=5) as w:
            client = boto3.client('s3')
            for e in enis:
                f_downloads[
                    w.submit(
                        eni_download_flows,
                        client, bucket,
                        log_prefix, start, end,
                        e['eni_id'], store_dir)] = e

            for f in as_completed(f_downloads):
                if f.exception():
                    log.warning(
                        "error processing eni %s download: %s",
                        eni_map[f_downloads[f]],
                        f.exception())
                    continue
                e = f_downloads[f]
                files[e['eni_id']] = f.result()

            ipset = {e['ip_address'] for e in enis}

            for eni_id, files in files.items():
                stream = eni_flow_stream(files, start, end)
                f_metrics[w.submit(
                    process_eni_metrics,
                    eni_map[eni_id], ipset,
                    stream,
                    start, end, period, sample_count,
                    resolver, sink)] = eni_id

            for f in as_completed(f_metrics):
                if f.exception():
                    log.warning(
                        "error processing eni %s download %s",
                        eni_map[f_metrics[f]],
                        f.exception())
                    continue

Response: Analyze flow log records for application and generate metrics per period


Instruction:
Can you generate the documentation for the following Python 3 function
def list_functions(self, prefix=None):
        return self.client.execute_command(
            'list',
            {'parent': "projects/{}/locations/{}".format(
                self.session.get_default_project(),
                self.region)}
        ).get('functions', [])

Response: List extant cloud functions.


Instruction:
Can you write a function in Python 3 where it
publishes the given function.

Response:
def publish(self, func):
        """publish the given function."""
        project = self.session.get_default_project()
        func_name = "projects/{}/locations/{}/functions/{}".format(
            project, self.region, func.name)
        func_info = self.get(func.name)
        source_url = None

        archive = func.get_archive()
        if not func_info or self._delta_source(archive, func_name):
            source_url = self._upload(archive, self.region)

        config = func.get_config()
        config['name'] = func_name
        if source_url:
            config['sourceUploadUrl'] = source_url

        # todo - we'll really need before() and after() for pre-provisioning of
        # resources (ie topic for function stream on create) and post provisioning (schedule
        # invocation of extant function).
        #
        # convergent event source creation
        for e in func.events:
            e.add(func)

        if func_info is None:
            log.info("creating function")
            response = self.client.execute_command(
                'create', {
                    'location': "projects/{}/locations/{}".format(
                        project, self.region),
                    'body': config})
        else:
            delta = delta_resource(func_info, config, ('httpsTrigger',))
            if not delta:
                response = None
            else:
                update_mask = ','.join(delta)
                log.info("updating function config %s", update_mask)
                response = self.client.execute_command(
                    'patch', {
                        'name': func_name,
                        'body': config,
                        'updateMask': update_mask})
        return response


Instruction:
Write a Python 3 function that can
get the details on a given function.

Response:
def get(self, func_name, qualifier=None):
        """Get the details on a given function."""
        project = self.session.get_default_project()
        func_name = "projects/{}/locations/{}/functions/{}".format(
            project, self.region, func_name)
        try:
            return self.client.execute_query('get', {'name': func_name})
        except errors.HttpError as e:
            if e.resp.status != 404:
                raise


Instruction:
Here you have a function in Python 3, explain what it does
def _upload(self, archive, region):
        # Generate source upload url
        url = self.client.execute_command(
            'generateUploadUrl',
            {'parent': 'projects/{}/locations/{}'.format(
                self.session.get_default_project(),
                region)}).get('uploadUrl')
        log.debug("uploading function code %s", url)
        http = self._get_http_client(self.client)
        headers, response = http.request(
            url, method='PUT',
            headers={
                'content-type': 'application/zip',
                'Content-Length': '%d' % archive.size,
                'x-goog-content-length-range': '0,104857600'
            },
            body=open(archive.path, 'rb')
        )
        log.info("function code uploaded")
        if headers['status'] != '200':
            raise RuntimeError("%s\n%s" % (headers, response))
        return url

Response: Upload function source and return source url


Instruction:
Make a summary of the following Python 3 code
def ensure_topic(self):
        client = self.session.client('pubsub', 'v1', 'projects.topics')
        topic = self.get_topic_param()
        try:
            client.execute_command('get', {'topic': topic})
        except HttpError as e:
            if e.resp.status != 404:
                raise
        else:
            return topic

        # bug in discovery doc.. apis say body must be empty but its required in the
        # discovery api for create.
        client.execute_command('create', {'name': topic, 'body': {}})
        return topic

Response: Verify the pub / sub topic exists. Returns the topic qualified name.


Instruction:
Write a Python 3 script for
ensuring the identities are in the iam role bindings for the topic.

Response:
def ensure_iam(self, publisher=None):
        """Ensure the given identities are in the iam role bindings for the topic.
        """
        topic = self.get_topic_param()
        client = self.session.client('pubsub', 'v1', 'projects.topics')
        policy = client.execute_command('getIamPolicy', {'resource': topic})
        policy.pop('etag')
        found = False
        for binding in policy.get('bindings', {}):
            if binding['role'] != 'roles/pubsub.publisher':
                continue
            if publisher in binding['members']:
                return
            found = binding

        if not found:
            policy.setdefault(
                'bindings', {'members': [publisher], 'role': 'roles/pubsub.publisher'})
        else:
            found['members'].append(publisher)

        client.execute_command('setIamPolicy', {'resource': topic, 'body': {'policy': policy}})


Instruction:
Create a Python 3 function to
get the parent container for the log sink

Response:
def get_parent(self, log_info):
        """Get the parent container for the log sink"""
        if self.data.get('scope', 'log') == 'log':
            if log_info.scope_type != 'projects':
                raise ValueError("Invalid log subscriber scope")
            parent = "%s/%s" % (log_info.scope_type, log_info.scope_id)
        elif self.data['scope'] == 'project':
            parent = 'projects/{}'.format(
                self.data.get('scope_id', self.session.get_default_project()))
        elif self.data['scope'] == 'organization':
            parent = 'organizations/{}'.format(self.data['scope_id'])
        elif self.data['scope'] == 'folder':
            parent = 'folders/{}'.format(self.data['scope_id'])
        elif self.data['scope'] == 'billing':
            parent = 'billingAccounts/{}'.format(self.data['scope_id'])
        else:
            raise ValueError(
                'invalid log subscriber scope %s' % (self.data))
        return parent


Instruction:
Make a summary of the following Python 3 code
def ensure_sink(self):
        topic_info = self.pubsub.ensure_topic()
        scope, sink_path, sink_info = self.get_sink(topic_info)
        client = self.session.client('logging', 'v2', '%s.sinks' % scope)
        try:
            sink = client.execute_command('get', {'sinkName': sink_path})
        except HttpError as e:
            if e.resp.status != 404:
                raise
            sink = client.execute_command('create', sink_info)
        else:
            delta = delta_resource(sink, sink_info['body'])
            if delta:
                sink_info['updateMask'] = ','.join(delta)
                sink_info['sinkName'] = sink_path
                sink_info.pop('parent')
                sink = client.execute_command('update', sink_info)
            else:
                return sink_path

        self.pubsub.ensure_iam(publisher=sink['writerIdentity'])
        return sink_path

Response: Ensure the log sink and its pub sub topic exist.


Instruction:
Given the following Python 3 function, write the documentation
def remove(self, func):
        if not self.data['name'].startswith(self.prefix):
            return
        parent = self.get_parent(self.get_log())
        _, sink_path, _ = self.get_sink()
        client = self.session.client(
            'logging', 'v2', '%s.sinks' % (parent.split('/', 1)[0]))
        try:
            client.execute_command(
                'delete', {'sinkName': sink_path})
        except HttpError as e:
            if e.resp.status != 404:
                raise

Response: Remove any provisioned log sink if auto created


Instruction:
Write a Python 3 function for
formatting log events and relay them to SNS

Response:
def process_log_event(event, context):
    """Format log events and relay via sns/email"""
    init()
    serialized = event['awslogs'].pop('data')
    data = json.loads(zlib.decompress(
        base64.b64decode(serialized), 16 + zlib.MAX_WBITS))

    # Fetch additional logs for context (20s window)
    timestamps = [e['timestamp'] for e in data['logEvents']]
    start = min(timestamps) - 1000 * 15
    end = max(timestamps) + 1000 * 5

    events = logs.get_log_events(
        logGroupName=data['logGroup'],
        logStreamName=data['logStream'],
        startTime=start,
        endTime=end,
        startFromHead=True)['events']

    message = [
        "An error was detected",
        "",
        "Log Group: %s" % data['logGroup'],
        "Log Stream: %s" % data['logStream'],
        "Log Owner: %s" % data['owner'],
        "",
        "Log Contents",
        ""]

    # We may get things delivered from log sub that are not in log events
    for evt in data['logEvents']:
        if evt not in events:
            events.append(evt)

    for evt in events:
        message.append(message_event(evt))
        message.append("")

    params = dict(
        TopicArn=config['topic'],
        Subject=config['subject'],
        Message='\n'.join(message))

    sns.publish(**params)


Instruction:
Given the following Python 3 function, write the documentation
def get_function(session_factory, name, role, sns_topic, log_groups,
                 subject="Lambda Error", pattern="Traceback"):
    """Lambda function provisioning.

    Self contained within the component, to allow for easier reuse.
    """

    # Lazy import to avoid runtime dependency
    from c7n.mu import (
        LambdaFunction, PythonPackageArchive, CloudWatchLogSubscription)

    config = dict(
        name=name,
        handler='logsub.process_log_event',
        runtime='python2.7',
        memory_size=512,
        timeout=15,
        role=role,
        description='Custodian Ops Error Notify',
        events=[
            CloudWatchLogSubscription(
                session_factory, log_groups, pattern)])

    archive = PythonPackageArchive()
    archive.add_py_file(__file__)
    archive.add_contents(
        'config.json', json.dumps({
            'topic': sns_topic,
            'subject': subject
        }))
    archive.close()

    return LambdaFunction(config, archive)

Response: Creates a Lambda function provisioning.


Instruction:
Create a Python 3 function to
match a given cwe event as cloudtrail with an api call

Response:
def match(cls, event):
        """Match a given cwe event as cloudtrail with an api call

        That has its information filled out.
        """
        if 'detail' not in event:
            return False
        if 'eventName' not in event['detail']:
            return False
        k = event['detail']['eventName']

        # We want callers to use a compiled expression, but want to avoid
        # initialization cost of doing it without cause. Not thread safe,
        # but usage context is lambda entry.
        if k in cls.trail_events:
            v = dict(cls.trail_events[k])
            if isinstance(v['ids'], six.string_types):
                v['ids'] = e = jmespath.compile('detail.%s' % v['ids'])
                cls.trail_events[k]['ids'] = e
            return v

        return False


Instruction:
Given the following Python 3 function, write the documentation
def get_trail_ids(cls, event, mode):
        resource_ids = ()
        event_name = event['detail']['eventName']
        event_source = event['detail']['eventSource']
        for e in mode.get('events', []):
            if not isinstance(e, dict):
                # Check if we have a short cut / alias
                info = CloudWatchEvents.match(event)
                if info:
                    return info['ids'].search(event)
                continue
            if event_name != e.get('event'):
                continue
            if event_source != e.get('source'):
                continue

            id_query = e.get('ids')
            if not id_query:
                raise ValueError("No id query configured")
            evt = event
            # be forgiving for users specifying with details or without
            if not id_query.startswith('detail.'):
                evt = event.get('detail', {})
            resource_ids = jmespath.search(id_query, evt)
            if resource_ids:
                break
        return resource_ids

Response: extract resources ids from a cloud trail event.


Instruction:
Implement a Python 3 function for
generating a c7n - org accounts config file using AWS Organizations

Response:
def main(role, ou, assume, profile, output, regions, active):
    """Generate a c7n-org accounts config file using AWS Organizations

    With c7n-org you can then run policies or arbitrary scripts across
    accounts.
    """

    session = get_session(assume, 'c7n-org', profile)
    client = session.client('organizations')
    accounts = []
    for path in ou:
        ou = get_ou_from_path(client, path)
        accounts.extend(get_accounts_for_ou(client, ou, active))

    results = []
    for a in accounts:
        tags = []
        path_parts = a['Path'].strip('/').split('/')
        for idx, _ in enumerate(path_parts):
            tags.append("path:/%s" % "/".join(path_parts[:idx + 1]))

        ainfo = {
            'account_id': a['Id'],
            'email': a['Email'],
            'name': a['Name'],
            'tags': tags,
            'role': role.format(**a)}
        if regions:
            ainfo['regions'] = regions
        results.append(ainfo)

    print(
        yaml.safe_dump(
            {'accounts': results},
            default_flow_style=False),
        file=output)


Instruction:
Can you create a Python 3 function that
downloads a traildb file for a given account day and region

Response:
def download(config, account, day, region, output):
    """Download a traildb file for a given account/day/region"""

    with open(config) as fh:
        config = yaml.safe_load(fh.read())

    jsonschema.validate(config, CONFIG_SCHEMA)

    found = None
    for info in config['accounts']:
        if info['name'] == account:
            found = info
            break

    if not found:
        log.info("Account %s not found", account)
        return

    s3 = boto3.client('s3')
    day = parse_date(day)

    key_data = dict(found)
    key_data['region'] = region
    key_data['date_fmt'] = "%s/%s/%s" % (
        day.year, day.month, day.day)
    key = config['key_template'] % key_data

    s3.download_file(found['bucket'], key, output + '.bz2')
    subprocess.check_call(["lbzip2", "-d", output + '.bz2'])


Instruction:
How would you implement a function in Python 3 that
shows the status of the last record time by account

Response:
def status(config):
    """time series lastest record time by account."""
    with open(config) as fh:
        config = yaml.safe_load(fh.read())
    jsonschema.validate(config, CONFIG_SCHEMA)
    last_index = get_incremental_starts(config, None)
    accounts = {}
    for (a, region), last in last_index.items():
        accounts.setdefault(a, {})[region] = last
    print(yaml.safe_dump(accounts, default_flow_style=False))


Instruction:
Here you have a function in Python 3, explain what it does
def index(config, start, end, incremental=False, concurrency=5, accounts=None,
          verbose=False):
    """index traildbs directly from s3 for multiple accounts.

    context: assumes a daily traildb file in s3 with key path
             specified by key_template in config file for each account
    """
    with open(config) as fh:
        config = yaml.safe_load(fh.read())
    jsonschema.validate(config, CONFIG_SCHEMA)

    if verbose:
        logging.root.setLevel(logging.DEBUG)
    log.info("tmpdir %s" % os.environ.get('TMPDIR'))

    with ProcessPoolExecutor(max_workers=concurrency) as w:
        futures = {}

        if incremental:
            account_starts = get_incremental_starts(config, start)
        else:
            account_starts = defaultdict(lambda : start) # NOQA E203

        for account in config.get('accounts'):
            if accounts and account['name'] not in accounts:
                continue
            for region in account.get('regions'):
                for d in get_date_range(
                        account_starts[(account['name'], region)], end):
                    i = bool(d.hour or d.minute)
                    p = (config, region, account, d, i)
                    futures[w.submit(index_account, *p)] = p

        for f in as_completed(futures):
            _, region, account, d, incremental = futures[f]

            result = f.result()
            if result is None:
                continue
            log.info(
                ("processed account:%(account)s day:%(day)s region:%(region)s "
                 "records:%(records)s time:%(time)0.2f db-date:%(db-date)s"
                 ) % result)

Response: index traildbs directly from s3 for multiple accounts


Instruction:
Can you create a Python 3 function that
returns a sqlite row factory that returns a dictionary

Response:
def dict_factory(cursor, row):
    """Returns a sqlite row factory that returns a dictionary"""
    d = {}
    for idx, col in enumerate(cursor.description):
        d[col[0]] = row[idx]
    return d


Instruction:
Can you generate a brief explanation for the following Python 3 code
def fetch_events(cursor, config, account_name):
    query = config['indexer'].get('query',
        'select * from events where user_agent glob \'*CloudCustodian*\'')

    for event in cursor.execute(query):
        event['account'] = account_name
        event['_index'] = config['indexer']['idx_name']
        event['_type'] = config['indexer'].get('idx_type', 'traildb')
        yield event

Response: Generator that returns the events that are available for the given account