Instruction:
Can you generate a brief explanation for the following Python 3 code
def filter_creation_date(groups, start, end):
    results = []
    for g in groups:
        created = datetime.fromtimestamp(g['creationTime'] / 1000.0)
        if created > end:
            continue
        if created > start:
            g['exportStart'] = created
        else:
            g['exportStart'] = start
        results.append(g)
    return results

Response: Filter log groups by their creation date.


Instruction:
Can you create a Python 3 function that
filters log groups where the last write was before the start date.

Response:
def filter_last_write(client, groups, start):
    """Filter log groups where the last write was before the start date.
    """
    retry = get_retry(('ThrottlingException',))

    def process_group(group_set):
        matched = []
        for g in group_set:
            streams = retry(
                client.describe_log_streams,
                logGroupName=g['logGroupName'],
                orderBy='LastEventTime',
                limit=1, descending=True)
            if not streams.get('logStreams'):
                continue
            stream = streams['logStreams'][0]
            if stream['storedBytes'] == 0 and datetime.fromtimestamp(
                    stream['creationTime'] / 1000) > start:
                matched.append(g)
            elif 'lastIngestionTime' in stream and datetime.fromtimestamp(
                    stream['lastIngestionTime'] / 1000) > start:
                matched.append(g)
        return matched

    results = []

    with ThreadPoolExecutor(max_workers=3) as w:
        futures = {}
        for group_set in chunks(groups, 10):
            futures[w.submit(process_group, group_set)] = group_set

        for f in as_completed(futures):
            if f.exception():
                log.error(
                    "Error processing groupset:%s error:%s",
                    group_set,
                    f.exception())
            results.extend(f.result())

    return results


Instruction:
Explain what the following Python 3 code does
def filter_extant_exports(client, bucket, prefix, days, start, end=None):
    end = end or datetime.now()
    # days = [start + timedelta(i) for i in range((end-start).days)]
    try:
        tag_set = client.get_object_tagging(Bucket=bucket, Key=prefix).get('TagSet', [])
    except ClientError as e:
        if e.response['Error']['Code'] != 'NoSuchKey':
            raise
        tag_set = []
    tags = {t['Key']: t['Value'] for t in tag_set}

    if 'LastExport' not in tags:
        return sorted(days)
    last_export = parse(tags['LastExport'])
    if last_export.tzinfo is None:
        last_export = last_export.replace(tzinfo=tzutc())
    return [d for d in sorted(days) if d > last_export]

Response: Filter days where the bucket already has extant export keys.


Instruction:
Explain what the following Python 3 code does
def access(config, region, accounts=()):
    config = validate.callback(config)
    accounts_report = []

    def check_access(account):
        accounts_report.append(account)
        session = get_session(account['role'], region)
        identity = session.client('sts').get_caller_identity()
        account['account_id'] = identity['Account']
        account.pop('groups')
        account.pop('role')
        client = session.client('iam')
        policy_arn = identity['Arn']
        if policy_arn.count('/') > 1:
            policy_arn = policy_arn.rsplit('/', 1)[0]
        if ':sts:' in policy_arn:
            policy_arn = policy_arn.replace(':sts', ':iam')
        if ':assumed-role' in policy_arn:
            policy_arn = policy_arn.replace(':assumed-role', ':role')
        evaluation = client.simulate_principal_policy(
            PolicySourceArn=policy_arn,
            ActionNames=['logs:CreateExportTask'])['EvaluationResults']
        account['access'] = evaluation[0]['EvalDecision']

    with ThreadPoolExecutor(max_workers=16) as w:
        futures = {}
        for account in config.get('accounts', ()):
            if accounts and account['name'] not in accounts:
                continue
            futures[w.submit(check_access, account)] = None
        for f in as_completed(futures):
            pass
    accounts_report.sort(key=operator.itemgetter('access'), reverse=True)
    print(tabulate(accounts_report, headers='keys'))

Response: Check iam permissions for log export access in each account


Instruction:
Here you have a function in Python 3, explain what it does
def size(config, accounts=(), day=None, group=None, human=True, region=None):
    config = validate.callback(config)
    destination = config.get('destination')
    client = boto3.Session().client('s3')
    day = parse(day)

    def export_size(client, account):
        paginator = client.get_paginator('list_objects_v2')
        count = 0
        size = 0
        session = get_session(account['role'], region)
        account_id = session.client('sts').get_caller_identity()['Account']
        prefix = destination.get('prefix', '').rstrip('/') + '/%s' % account_id
        prefix = "%s/%s/%s" % (prefix, group, day.strftime("%Y/%m/%d"))
        account['account_id'] = account_id
        for page in paginator.paginate(
                Bucket=destination['bucket'],
                Prefix=prefix):
            for k in page.get('Contents', ()):
                size += k['Size']
                count += 1
        return (count, size)

    total_size = 0
    accounts_report = []
    logging.getLogger('botocore').setLevel(logging.ERROR)
    with ThreadPoolExecutor(max_workers=16) as w:
        futures = {}
        for account in config.get('accounts'):
            if accounts and account['name'] not in accounts:
                continue
            futures[w.submit(export_size, client, account)] = account

        for f in as_completed(futures):
            account = futures[f]
            count, size = f.result()
            account.pop('role')
            account.pop('groups')
            total_size += size
            if human:
                account['size'] = GetHumanSize(size)
            else:
                account['size'] = size
            account['count'] = count
            accounts_report.append(account)

    accounts_report.sort(key=operator.itemgetter('count'), reverse=True)
    print(tabulate(accounts_report, headers='keys'))
    log.info("total size:%s", GetHumanSize(total_size))

Response: Returns a list of exported records for a given day.


Instruction:
Here you have a function in Python 3, explain what it does
def sync(config, group, accounts=(), dryrun=False, region=None):
    config = validate.callback(config)
    destination = config.get('destination')
    client = boto3.Session().client('s3')

    for account in config.get('accounts', ()):
        if accounts and account['name'] not in accounts:
            continue

        session = get_session(account['role'], region)
        account_id = session.client('sts').get_caller_identity()['Account']
        prefix = destination.get('prefix', '').rstrip('/') + '/%s' % account_id
        prefix = "%s/%s" % (prefix, group)

        exports = get_exports(client, destination['bucket'], prefix + "/")

        role = account.pop('role')
        if isinstance(role, six.string_types):
            account['account_id'] = role.split(':')[4]
        else:
            account['account_id'] = role[-1].split(':')[4]
        account.pop('groups')

        if exports:
            last_export = exports.pop()
            account['export'] = last_export
        else:
            account['export'] = 'missing'
            last_export = None
        try:
            tag_set = client.get_object_tagging(
                Bucket=destination['bucket'], Key=prefix).get('TagSet', [])
        except ClientError:
            tag_set = []

        tags = {t['Key']: t['Value'] for t in tag_set}
        tagged_last_export = None

        if 'LastExport' in tags:
            le = parse(tags['LastExport'])
            tagged_last_export = (le.year, le.month, le.day)
            account['sync'] = tagged_last_export
        else:
            account['sync'] = account['export'] != 'missing' and 'sync' or 'missing'

        if last_export is None:
            continue

        if tagged_last_export == last_export or account['export'] == 'missing':
            continue

        if dryrun:
            continue

        client.put_object(
            Bucket=destination['bucket'],
            Key=prefix,
            Body=json.dumps({}),
            ACL="bucket-owner-full-control",
            ServerSideEncryption="AES256")

        export_time = datetime.now().replace(tzinfo=tzlocal()).astimezone(tzutc())
        export_time = export_time.replace(
            year=last_export[0], month=last_export[1], day=last_export[2],
            minute=0, second=0, microsecond=0, hour=0)
        client.put_object_tagging(
            Bucket=destination['bucket'], Key=prefix,
            Tagging={
                'TagSet': [{
                    'Key': 'LastExport',
                    'Value': export_time.isoformat()}]})

    accounts_report = []
    for a in config.get('accounts'):
        if accounts and a['name'] not in accounts:
            continue
        if isinstance(a['sync'], tuple):
            a['sync'] = "%s/%s/%s" % (a['sync'])
        if isinstance(a['export'], tuple):
            a['export'] = "%s/%s/%s" % (a['export'])
        accounts_report.append(a)

    accounts_report.sort(key=operator.itemgetter('export'), reverse=True)
    print(tabulate(accounts_report, headers='keys'))

Response: Sync the last recorded export to actual


Instruction:
Make a summary of the following Python 3 code
def status(config, group, accounts=(), region=None):
    config = validate.callback(config)
    destination = config.get('destination')
    client = boto3.Session().client('s3')

    for account in config.get('accounts', ()):
        if accounts and account['name'] not in accounts:
            continue

        session = get_session(account['role'], region)
        account_id = session.client('sts').get_caller_identity()['Account']
        prefix = destination.get('prefix', '').rstrip('/') + '/%s' % account_id
        prefix = "%s/flow-log" % prefix

        role = account.pop('role')
        if isinstance(role, six.string_types):
            account['account_id'] = role.split(':')[4]
        else:
            account['account_id'] = role[-1].split(':')[4]

        account.pop('groups')

        try:
            tag_set = client.get_object_tagging(
                Bucket=destination['bucket'], Key=prefix).get('TagSet', [])
        except ClientError:
            account['export'] = 'missing'
            continue
        tags = {t['Key']: t['Value'] for t in tag_set}

        if 'LastExport' not in tags:
            account['export'] = 'empty'
        else:
            last_export = parse(tags['LastExport'])
            account['export'] = last_export.strftime('%Y/%m/%d')

    accounts = [a for a in config.get('accounts') if a in accounts or not accounts]
    accounts.sort(key=operator.itemgetter('export'), reverse=True)
    print(tabulate(accounts, headers='keys'))

Response: report current export state status


Instruction:
Can you create a Python 3 function that
finds exports for a given account

Response:
def get_exports(client, bucket, prefix, latest=True):
    """Find exports for a given account
    """
    keys = client.list_objects_v2(
        Bucket=bucket, Prefix=prefix, Delimiter='/').get('CommonPrefixes', [])
    found = []
    years = []
    for y in keys:
        part = y['Prefix'].rsplit('/', 2)[-2]
        if not part.isdigit():
            continue
        year = int(part)
        years.append(year)

    if not years:
        return []

    years.sort(reverse=True)
    if latest:
        years = [years[0]]

    for y in years:
        keys = client.list_objects_v2(
            Bucket=bucket, Prefix="%s/%d/" % (prefix.strip('/'), y),
            Delimiter='/').get('CommonPrefixes', [])
        months = []
        for m in keys:
            part = m['Prefix'].rsplit('/', 2)[-2]
            if not part.isdigit():
                continue
            month = int(part)
            date_key = (y, month)
            months.append(month)
        months.sort(reverse=True)
        if not months:
            continue
        if latest:
            months = [months[0]]
        for m in months:
            keys = client.list_objects_v2(
                Bucket=bucket, Prefix="%s/%d/%s/" % (
                    prefix.strip('/'), y, ('%d' % m).rjust(2, '0')),
                Delimiter='/').get('CommonPrefixes', [])
            for d in keys:
                part = d['Prefix'].rsplit('/', 2)[-2]
                if not part.isdigit():
                    continue
                day = int(part)
                date_key = (y, m, day)
                found.append(date_key)
    found.sort(reverse=True)
    if latest:
        found = [found[0]]
    return found


Instruction:
Can you implement a function in Python 3 that
exports a given log group to s3

Response:
def export(group, bucket, prefix, start, end, role, poll_period=120,
           session=None, name="", region=None):
    """export a given log group to s3"""
    start = start and isinstance(start, six.string_types) and parse(start) or start
    end = (end and isinstance(start, six.string_types) and
           parse(end) or end or datetime.now())
    start = start.replace(tzinfo=tzlocal()).astimezone(tzutc())
    end = end.replace(tzinfo=tzlocal()).astimezone(tzutc())

    if session is None:
        session = get_session(role, region)

    client = session.client('logs')

    paginator = client.get_paginator('describe_log_groups')
    for p in paginator.paginate():
        found = False
        for _group in p['logGroups']:
            if _group['logGroupName'] == group:
                group = _group
                found = True
                break
        if found:
            break

    if not found:
        raise ValueError("Log group %s not found." % group)

    if prefix:
        prefix = "%s/%s" % (prefix.rstrip('/'), group['logGroupName'].strip('/'))
    else:
        prefix = group['logGroupName']

    named_group = "%s:%s" % (name, group['logGroupName'])
    log.info(
        "Log exporting group:%s start:%s end:%s bucket:%s prefix:%s size:%s",
        named_group,
        start.strftime('%Y/%m/%d'),
        end.strftime('%Y/%m/%d'),
        bucket,
        prefix,
        group['storedBytes'])

    t = time.time()
    days = [(
        start + timedelta(i)).replace(minute=0, hour=0, second=0, microsecond=0)
        for i in range((end - start).days)]
    day_count = len(days)
    s3 = boto3.Session().client('s3')
    days = filter_extant_exports(s3, bucket, prefix, days, start, end)

    log.info("Group:%s filtering s3 extant keys from %d to %d start:%s end:%s",
             named_group, day_count, len(days),
             days[0] if days else '', days[-1] if days else '')
    t = time.time()

    retry = get_retry(('SlowDown',))

    for idx, d in enumerate(days):
        date = d.replace(minute=0, microsecond=0, hour=0)
        export_prefix = "%s%s" % (prefix, date.strftime("/%Y/%m/%d"))
        params = {
            'taskName': "%s-%s" % ("c7n-log-exporter",
                                   date.strftime("%Y-%m-%d")),
            'logGroupName': group['logGroupName'],
            'fromTime': int(time.mktime(
                date.replace(
                    minute=0, microsecond=0, hour=0).timetuple()) * 1000),
            'to': int(time.mktime(
                date.replace(
                    minute=59, hour=23, microsecond=0).timetuple()) * 1000),
            'destination': bucket,
            'destinationPrefix': export_prefix
        }

        # if stream_prefix:
        #    params['logStreamPrefix'] = stream_prefix
        try:
            s3.head_object(Bucket=bucket, Key=prefix)
        except ClientError as e:
            if e.response['Error']['Code'] != '404':  # Not Found
                raise
            s3.put_object(
                Bucket=bucket,
                Key=prefix,
                Body=json.dumps({}),
                ACL="bucket-owner-full-control",
                ServerSideEncryption="AES256")

        t = time.time()
        counter = 0
        while True:
            counter += 1
            try:
                result = client.create_export_task(**params)
            except ClientError as e:
                if e.response['Error']['Code'] == 'LimitExceededException':
                    time.sleep(poll_period)
                    # log every 30m of export waiting
                    if counter % 6 == 0:
                        log.debug(
                            "group:%s day:%s waiting for %0.2f minutes",
                            named_group, d.strftime('%Y-%m-%d'),
                            (counter * poll_period) / 60.0)
                    continue
                raise
            retry(
                s3.put_object_tagging,
                Bucket=bucket, Key=prefix,
                Tagging={
                    'TagSet': [{
                        'Key': 'LastExport',
                        'Value': d.isoformat()}]})
            break

        log.info(
            "Log export time:%0.2f group:%s day:%s bucket:%s prefix:%s task:%s",
            time.time() - t,
            named_group,
            d.strftime("%Y-%m-%d"),
            bucket,
            params['destinationPrefix'],
            result['taskId'])

    log.info(
        ("Exported log group:%s time:%0.2f days:%d start:%s"
         " end:%s bucket:%s prefix:%s"),
        named_group,
        time.time() - t,
        len(days),
        start.strftime('%Y/%m/%d'),
        end.strftime('%Y/%m/%d'),
        bucket,
        prefix)


Instruction:
Write a Python 3 function that can
process a single log event and relay it to sentry

Response:
def process_log_event(event, context):
    """Lambda Entrypoint - Log Subscriber

    Format log events and relay to sentry (direct or sqs)
    """
    init()
    # Grab the actual error log payload
    serialized = event['awslogs'].pop('data')
    data = json.loads(zlib.decompress(
        base64.b64decode(serialized), 16 + zlib.MAX_WBITS))
    msg = get_sentry_message(config, data)
    if msg is None:
        return
    if config['sentry_dsn']:
        # Deliver directly to sentry
        send_sentry_message(config['sentry_dsn'], msg)
    elif config['sentry_sqs']:
        # Delivery indirectly via sqs
        sqs.send_message(
            QueueUrl=config['sentry_sqs'])


Instruction:
Can you write a function in Python 3 where it
processes the log group and sends the messages to the Sentry server.

Response:
def process_log_group(config):
    """CLI - Replay / Index
    """

    from c7n.credentials import SessionFactory
    factory = SessionFactory(
        config.region, config.profile, assume_role=config.role)
    session = factory()
    client = session.client('logs')

    params = dict(logGroupName=config.log_group,
                  filterPattern='Traceback', interleaved=True)
    if config.log_streams:
        params['logStreamNames'] = config.log_streams

    if config.start:
        params['startTime'] = int(time.mktime(
            parse_date(config.start).replace(
                hour=0, minute=0, second=0, microsecond=0).timetuple()) * 1000)
    if config.end:
        params['endTime'] = int(time.mktime(
            parse_date(config.end).replace(
                hour=0, minute=0, second=0, microsecond=0).timetuple()) * 1000)

    settings = dict(account_id=config.account_id,
                    account_name=config.account_name)
    paginator = client.get_paginator('filter_log_events')

    event_count = 0
    log.debug("Querying log events with %s", params)
    for p in paginator.paginate(**params):
        # log.debug("Searched streams\n %s", ", ".join(
        #    [s['logStreamName'] for s in p['searchedLogStreams']]))
        for e in p['events']:
            event_count += 1
            msg = get_sentry_message(
                settings, {'logEvents': [e],
                           'logStream': e['logStreamName'],
                           'logGroup': config.log_group}, client)
            if msg is None:
                continue
            send_sentry_message(config.sentry_dsn, msg)

    if event_count > 0:
        log.info("Processed %s %d error events", config.account_name, event_count)


Instruction:
How would you explain what the following Python 3 function does
def parse_traceback(msg, site_path="site-packages", in_app_prefix="c7n"):

    data = {}
    lines = list(filter(None, msg.split('\n')))
    data['frames'] = []
    err_ctx = None

    for l in lines[1:-1]:
        l = l.strip() # noqa E741
        if l.startswith('Traceback'):
            continue
        elif l.startswith('File'):
            abs_path, lineno, function = l.split(',', 3)
            abs_path = abs_path[abs_path.find('"'):-1]
            f_path = abs_path[abs_path.find(site_path) + len(site_path) + 1:]
            module = f_path[:f_path.find('.')].replace('/', '.').strip('.')
            lineno = int(lineno.strip().split()[1])
            function = function.strip().split()[-1]
            err_ctx = dict(lineno=lineno,
                           abs_path=abs_path,
                           function=function,
                           filename=f_path,
                           module=module)
            if module.startswith(in_app_prefix):
                err_ctx['in_app'] = True
        elif err_ctx is not None:
            err_ctx['context_line'] = l
            data['frames'].append(err_ctx)
            err_ctx = None

    return lines[0], {
        'type': lines[-1].strip().split(':')[0],
        'value': lines[-1].strip().split(':', 1)[1].strip(),
        'module': data['frames'][-1]['module'],
        'stacktrace': data}

Response: Extract a sentry traceback structure from a python formatted traceback string per python stdlib traceback. print_exc


Instruction:
Can you write a function in Python 3 where it
gets a Lambda function provisioning.

Response:
def get_function(session_factory, name, handler, runtime, role,
                 log_groups,
                 project, account_name, account_id,
                 sentry_dsn,
                 pattern="Traceback"):
    """Lambda function provisioning.

    Self contained within the component, to allow for easier reuse.
    """
    # Lazy import to avoid runtime dependency
    from c7n.mu import (
        LambdaFunction, PythonPackageArchive, CloudWatchLogSubscription)

    config = dict(
        name=name,
        handler=handler,
        runtime=runtime,
        memory_size=512,
        timeout=15,
        role=role,
        description='Custodian Sentry Relay',
        events=[
            CloudWatchLogSubscription(
                session_factory, log_groups, pattern)])

    archive = PythonPackageArchive('c7n_sentry')
    archive.add_contents(
        'config.json', json.dumps({
            'project': project,
            'account_name': account_name,
            'account_id': account_id,
            'sentry_dsn': sentry_dsn,
        }))
    archive.add_contents(
        'handler.py',
        'from c7n_sentry.c7nsentry import process_log_event'
    )
    archive.close()

    return LambdaFunction(config, archive)


Instruction:
Write a Python 3 function that can
break an iterable into lists of size

Response:
def chunks(iterable, size=50):
    """Break an iterable into lists of size"""
    batch = []
    for n in iterable:
        batch.append(n)
        if len(batch) % size == 0:
            yield batch
            batch = []
    if batch:
        yield batch


Instruction:
How would you explain what the following Python 3 function does
def list_app_resources(
        app, env, resources, cmdb, start, end, tz):
    """Analyze flow log records for application and generate metrics per period"""
    logging.basicConfig(level=logging.INFO)
    start, end = get_dates(start, end, tz)

    all_resources = []
    for rtype_name in resources:
        rtype = Resource.get_type(rtype_name)
        resources = rtype.get_resources(cmdb, start, end, app, env)
        all_resources.extend(resources)
    print(json.dumps(all_resources, indent=2))

Response: Analyze flow log records for application and generate metrics per period


Instruction:
Create a Python 3 function for
loading resources into resource database.

Response:
def load_resources(bucket, prefix, region, account_config, accounts,
                   assume, start, end, resources, store, db, verbose, debug):
    """load resources into resource database."""
    logging.basicConfig(level=(verbose and logging.DEBUG or logging.INFO))
    logging.getLogger('botocore').setLevel(logging.WARNING)
    logging.getLogger('s3transfer').setLevel(logging.WARNING)
    start = date_parse(start)
    end = date_parse(end)

    if not resources:
        resources = ['NetworkInterface', 'Instance', 'LoadBalancer']

    account_map = {}
    data = yaml.safe_load(account_config.read())
    for a in data.get('accounts', ()):
        if accounts and (a['name'] in accounts or a['account_id'] in accounts):
            account_map[a['account_id']] = a
        elif not accounts:
            account_map[a['account_id']] = a
    account_ids = list(account_map)

    executor = ProcessPoolExecutor
    if debug:
        from c7n.executor import MainThreadExecutor
        MainThreadExecutor.c7n_async = False
        executor = MainThreadExecutor

    stats = Counter()
    t = time.time()
    with executor(max_workers=multiprocessing.cpu_count()) as w:
        futures = {}
        for a in account_ids:
            for r in resources:
                futures[w.submit(
                    process_account_resources, a, bucket, prefix,
                    region, store, start, end, r)] = (a, r)

        indexer = RESOURCE_FILE_INDEXERS[r]
        for f in as_completed(futures):
            a, r = futures[f]
            if f.exception():
                log.error("account:%s error:%s", a, f.exception())
                continue
            files, dl_stats = f.result()
            idx_stats = indexer(db, resource_config_iter(files))
            log.info(
                "loaded account:%s files:%d bytes:%s events:%d resources:%d idx-time:%d dl-time:%d",
                account_map[a]['name'], len(files),
                human_size(dl_stats['DownloadSize'] + dl_stats['CacheSize']),
                idx_stats['Records'],
                idx_stats['RowCount'],
                idx_stats['IndexTime'],
                dl_stats['FetchTime'])
            stats.update(dl_stats)
            stats.update(idx_stats)
    log.info("Loaded %d resources across %d accounts in %0.2f",
             stats['RowCount'], len(account_ids), time.time() - t)


Instruction:
Can you generate the documentation for the following Python 3 function
def load_plugins(self):
        try:
            from pkg_resources import iter_entry_points
        except ImportError:
            return
        for ep in iter_entry_points(group="custodian.%s" % self.plugin_type):
            f = ep.load()
            f()

Response:
Load external plugins.

        Custodian is intended to interact with internal and external systems
        that are not suitable for embedding into the custodian code base.


Instruction:
Create a Python 3 function for
submitting a function for serialized execution on sqs

Response:
def submit(self, func, *args, **kwargs):
        """Submit a function for serialized execution on sqs
        """
        self.op_sequence += 1
        self.sqs.send_message(
            QueueUrl=self.map_queue,
            MessageBody=utils.dumps({'args': args, 'kwargs': kwargs}),
            MessageAttributes={
                'sequence_id': {
                    'StringValue': str(self.op_sequence),
                    'DataType': 'Number'},
                'op': {
                    'StringValue': named(func),
                    'DataType': 'String',
                },
                'ser': {
                    'StringValue': 'json',
                    'DataType': 'String'}}
        )

        self.futures[self.op_sequence] = f = SQSFuture(
            self.op_sequence)
        return f


Instruction:
Implement a function in Python 3 to
gather the results from the queue and set them as result of the next concurrent process.

Response:
def gather(self):
        """Fetch results from separate queue
        """
        limit = self.op_sequence - self.op_sequence_start
        results = MessageIterator(self.sqs, self.reduce_queue, limit)
        for m in results:
            # sequence_id from above
            msg_id = int(m['MessageAttributes']['sequence_id']['StringValue'])
            if (not msg_id > self.op_sequence_start or not msg_id <= self.op_sequence or
            msg_id not in self.futures):
                raise RuntimeError(
                    "Concurrent queue user from different "
                    "process or previous results")
            f = self.futures[msg_id]
            f.set_result(m)
            results.ack(m)


Instruction:
Create a Python 3 function for
normalizing tag format on ecs resources to match common aws format

Response:
def ecs_tag_normalize(resources):
    """normalize tag format on ecs resources to match common aws format."""
    for r in resources:
        if 'tags' in r:
            r['Tags'] = [{'Key': t['key'], 'Value': t['value']} for t in r['tags']]
            r.pop('tags')


Instruction:
Write a Python 3 function that can
retrieve ecs resources for serverless policies or related resources

Response:
def get_resources(self, ids, cache=True):
        """Retrieve ecs resources for serverless policies or related resources

        Requires arns in new format.
        https://docs.aws.amazon.com/AmazonECS/latest/userguide/ecs-resource-ids.html
        """
        cluster_resources = {}
        for i in ids:
            _, ident = i.rsplit(':', 1)
            parts = ident.split('/', 2)
            if len(parts) != 3:
                raise PolicyExecutionError("New format ecs arn required")
            cluster_resources.setdefault(parts[1], []).append(parts[2])

        results = []
        client = local_session(self.manager.session_factory).client('ecs')
        for cid, resource_ids in cluster_resources.items():
            results.extend(
                self.process_cluster_resources(client, cid, resource_ids))
        return results


Instruction:
Create a Python 3 function to
resource types used by the collection.

Response:
def resource_types(self):
        """resource types used by the collection."""
        rtypes = set()
        for p in self.policies:
            rtypes.add(p.resource_type)
        return rtypes


Instruction:
Make a summary of the following Python 3 code
def get_metrics(self, start, end, period):
        values = {}
        default_dimensions = {
            'Policy': self.policy.name, 'ResType': self.policy.resource_type,
            'Scope': 'Policy'}

        metrics = list(self.POLICY_METRICS)

        # Support action, and filter custom metrics
        for el in itertools.chain(
                self.policy.resource_manager.actions,
                self.policy.resource_manager.filters):
            if el.metrics:
                metrics.extend(el.metrics)

        session = utils.local_session(self.policy.session_factory)
        client = session.client('cloudwatch')

        for m in metrics:
            if isinstance(m, six.string_types):
                dimensions = default_dimensions
            else:
                m, m_dimensions = m
                dimensions = dict(default_dimensions)
                dimensions.update(m_dimensions)
            results = client.get_metric_statistics(
                Namespace=DEFAULT_NAMESPACE,
                Dimensions=[
                    {'Name': k, 'Value': v} for k, v
                    in dimensions.items()],
                Statistics=['Sum', 'Average'],
                StartTime=start,
                EndTime=end,
                Period=period,
                MetricName=m)
            values[m] = results['Datapoints']
        return values

Response: Retrieve any associated metrics for the policy.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def run(self, event, lambda_context):
        from c7n.actions import EventAction

        mode = self.policy.data.get('mode', {})
        if not bool(mode.get("log", True)):
            root = logging.getLogger()
            map(root.removeHandler, root.handlers[:])
            root.handlers = [logging.NullHandler()]

        resources = self.resolve_resources(event)
        if not resources:
            return resources
        resources = self.policy.resource_manager.filter_resources(
            resources, event)

        if 'debug' in event:
            self.policy.log.info("Filtered resources %d" % len(resources))

        if not resources:
            self.policy.log.info(
                "policy: %s resources: %s no resources matched" % (
                    self.policy.name, self.policy.resource_type))
            return

        with self.policy.ctx:
            self.policy.ctx.metrics.put_metric(
                'ResourceCount', len(resources), 'Count', Scope="Policy",
                buffer=False)

            if 'debug' in event:
                self.policy.log.info(
                    "Invoking actions %s", self.policy.resource_manager.actions)

            self.policy._write_file(
                'resources.json', utils.dumps(resources, indent=2))

            for action in self.policy.resource_manager.actions:
                self.policy.log.info(
                    "policy: %s invoking action: %s resources: %d",
                    self.policy.name, action.name, len(resources))
                if isinstance(action, EventAction):
                    results = action.process(resources, event)
                else:
                    results = action.process(resources)
                self.policy._write_file(
                    "action-%s" % action.name, utils.dumps(results))
        return resources

Response: Run policy in push mode against given event.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_variables(self, variables=None):
        # Global policy variable expansion, we have to carry forward on
        # various filter/action local vocabularies. Where possible defer
        # by using a format string.
        #
        # See https://github.com/capitalone/cloud-custodian/issues/2330
        if not variables:
            variables = {}

        if 'mode' in self.data:
            if 'role' in self.data['mode'] and not self.data['mode']['role'].startswith("arn:aws"):
                self.data['mode']['role'] = "arn:aws:iam::%s:role/%s" % \
                                            (self.options.account_id, self.data['mode']['role'])

        variables.update({
            # standard runtime variables for interpolation
            'account': '{account}',
            'account_id': self.options.account_id,
            'region': self.options.region,
            # non-standard runtime variables from local filter/action vocabularies
            #
            # notify action
            'policy': self.data,
            'event': '{event}',
            # mark for op action
            'op': '{op}',
            'action_date': '{action_date}',
            # tag action pyformat-date handling
            'now': utils.FormatDate(datetime.utcnow()),
            # account increase limit action
            'service': '{service}',
            # s3 set logging action :-( see if we can revisit this one.
            'bucket_region': '{bucket_region}',
            'bucket_name': '{bucket_name}',
            'source_bucket_name': '{source_bucket_name}',
            'target_bucket_name': '{target_bucket_name}',
            'target_prefix': '{target_prefix}',
            'LoadBalancerName': '{LoadBalancerName}'
        })
        return variables

Response: Get runtime variables for policy interpolation.


Instruction:
Given the following Python 3 function, write the documentation
def expand_variables(self, variables):
        # format string values returns a copy
        updated = utils.format_string_values(self.data, **variables)

        # Several keys should only be expanded at runtime, perserve them.
        if 'member-role' in updated.get('mode', {}):
            updated['mode']['member-role'] = self.data['mode']['member-role']

        # Update ourselves in place
        self.data = updated
        # Reload filters/actions using updated data, we keep a reference
        # for some compatiblity preservation work.
        m = self.resource_manager
        self.resource_manager = self.load_resource_manager()

        # XXX: Compatiblity hack
        # Preserve notify action subject lines which support
        # embedded jinja2 as a passthrough to the mailer.
        for old_a, new_a in zip(m.actions, self.resource_manager.actions):
            if old_a.type == 'notify' and 'subject' in old_a.data:
                new_a.data['subject'] = old_a.data['subject']

Response: Expand variables in the policy data.


Instruction:
Write a Python 3 script to
get permissions needed by this policy

Response:
def get_permissions(self):
        """get permissions needed by this policy"""
        permissions = set()
        permissions.update(self.resource_manager.get_permissions())
        for f in self.resource_manager.filters:
            permissions.update(f.get_permissions())
        for a in self.resource_manager.actions:
            permissions.update(a.get_permissions())
        return permissions


Instruction:
Implement a Python 3 function for
handling various client side errors when describing snapshots

Response:
def extract_bad_snapshot(e):
        """Handle various client side errors when describing snapshots"""
        msg = e.response['Error']['Message']
        error = e.response['Error']['Code']
        e_snap_id = None
        if error == 'InvalidSnapshot.NotFound':
            e_snap_id = msg[msg.find("'") + 1:msg.rfind("'")]
            log.warning("Snapshot not found %s" % e_snap_id)
        elif error == 'InvalidSnapshotID.Malformed':
            e_snap_id = msg[msg.find('"') + 1:msg.rfind('"')]
            log.warning("Snapshot id malformed %s" % e_snap_id)
        return e_snap_id


Instruction:
Here you have a function in Python 3, explain what it does
def assumed_session(role_arn, session_name, session=None, region=None, external_id=None):
    if session is None:
        session = Session()

    retry = get_retry(('Throttling',))

    def refresh():

        parameters = {"RoleArn": role_arn, "RoleSessionName": session_name}

        if external_id is not None:
            parameters['ExternalId'] = external_id

        credentials = retry(
            session.client('sts').assume_role, **parameters)['Credentials']
        return dict(
            access_key=credentials['AccessKeyId'],
            secret_key=credentials['SecretAccessKey'],
            token=credentials['SessionToken'],
            # Silly that we basically stringify so it can be parsed again
            expiry_time=credentials['Expiration'].isoformat())

    session_credentials = RefreshableCredentials.create_from_metadata(
        metadata=refresh(),
        refresh_using=refresh,
        method='sts-assume-role')

    # so dirty.. it hurts, no clean way to set this outside of the
    # internals poke. There's some work upstream on making this nicer
    # but its pretty baroque as well with upstream support.
    # https://github.com/boto/boto3/issues/443
    # https://github.com/boto/botocore/issues/761

    s = get_session()
    s._credentials = session_credentials
    if region is None:
        region = s.get_config_variable('region') or 'us-east-1'
    s.set_config_variable('region', region)
    return Session(botocore_session=s)

Response: This function returns a boto3. Session object with the credentials for the given role and session.


Instruction:
Create a Python 3 function to
do the resource tag schedule and policy match the current time?

Response:
def process_resource_schedule(self, i, value, time_type):
        """Does the resource tag schedule and policy match the current time."""
        rid = i[self.id_key]
        # this is to normalize trailing semicolons which when done allows
        # dateutil.parser.parse to process: value='off=(m-f,1);' properly.
        # before this normalization, some cases would silently fail.
        value = ';'.join(filter(None, value.split(';')))
        if self.parser.has_resource_schedule(value, time_type):
            schedule = self.parser.parse(value)
        elif self.parser.keys_are_valid(value):
            # respect timezone from tag
            raw_data = self.parser.raw_data(value)
            if 'tz' in raw_data:
                schedule = dict(self.default_schedule)
                schedule['tz'] = raw_data['tz']
            else:
                schedule = self.default_schedule
        else:
            schedule = None
        if schedule is None:
            log.warning(
                "Invalid schedule on resource:%s value:%s", rid, value)
            self.parse_errors.append((rid, value))
            return False
        tz = self.get_tz(schedule['tz'])
        if not tz:
            log.warning(
                "Could not resolve tz on resource:%s value:%s", rid, value)
            self.parse_errors.append((rid, value))
            return False
        now = datetime.datetime.now(tz).replace(
            minute=0, second=0, microsecond=0)
        now_str = now.strftime("%Y-%m-%d")
        if 'skip-days-from' in self.data:
            values = ValuesFrom(self.data['skip-days-from'], self.manager)
            self.skip_days = values.get_values()
        else:
            self.skip_days = self.data.get('skip-days', [])
        if now_str in self.skip_days:
            return False
        return self.match(now, schedule)


Instruction:
Write a Python 3 function that can
get the resource s tag value specifying its schedule.

Response:
def get_tag_value(self, i):
        """Get the resource's tag value specifying its schedule."""
        # Look for the tag, Normalize tag key and tag value
        found = False
        for t in i.get('Tags', ()):
            if t['Key'].lower() == self.tag_key:
                found = t['Value']
                break
        if found is False:
            return False
        # enforce utf8, or do translate tables via unicode ord mapping
        value = found.lower().encode('utf8').decode('utf8')
        # Some folks seem to be interpreting the docs quote marks as
        # literal for values.
        value = value.strip("'").strip('"')
        return value


Instruction:
Can you generate a brief explanation for the following Python 3 code
def raw_data(tag_value):
        data = {}
        pieces = []
        for p in tag_value.split(' '):
            pieces.extend(p.split(';'))
        # parse components
        for piece in pieces:
            kv = piece.split('=')
            # components must by key=value
            if not len(kv) == 2:
                continue
            key, value = kv
            data[key] = value
        return data

Response:
convert the tag value to a dictionary taking values as is

        This method name and purpose are opaque... and not true.


Instruction:
Can you tell what is the following Python 3 function doing
def keys_are_valid(self, tag_value):
        for key in ScheduleParser.raw_data(tag_value):
            if key not in ('on', 'off', 'tz'):
                return False
        return True

Response: test that provided tag keys are valid


Instruction:
Can you generate a brief explanation for the following Python 3 code
def resources_gc_prefix(options, policy_config, policy_collection):

    # Classify policies by region
    policy_regions = {}
    for p in policy_collection:
        if p.execution_mode == 'poll':
            continue
        policy_regions.setdefault(p.options.region, []).append(p)

    regions = get_gc_regions(options.regions)
    for r in regions:
        region_gc(options, r, policy_config, policy_regions.get(r, []))

Response: Garbage collect old custodian policies based on prefix.


Instruction:
Write a Python 3 function for
getting a boto3 session potentially cross account sts

Response:
def get_session(account_info):
    """Get a boto3 sesssion potentially cross account sts assumed

    assumed sessions are automatically refreshed.
    """
    s = getattr(CONN_CACHE, '%s-session' % account_info['name'], None)
    if s is not None:
        return s
    if account_info.get('role'):
        s = assumed_session(account_info['role'], SESSION_NAME)
    else:
        s = boto3.Session()
    setattr(CONN_CACHE, '%s-session' % account_info['name'], s)
    return s


Instruction:
Can you write a function in Python 3 where it
bulks invoke a function via queues ridge Uses internal implementation details of rq. ridge

Response:
def bulk_invoke(func, args, nargs):
    """Bulk invoke a function via queues

    Uses internal implementation details of rq.
    """
    # for comparison, simplest thing that works
    # for i in nargs:
    #    argv = list(args)
    #    argv.append(i)
    #    func.delay(*argv)

    # some variances between cpy and pypy, sniff detect
    for closure in func.delay.func_closure:
        if getattr(closure.cell_contents, 'queue', None):
            ctx = closure.cell_contents
            break
    q = Queue(ctx.queue, connection=connection)
    argv = list(args)
    argv.append(None)
    job = Job.create(
        func, args=argv, connection=connection,
        description="bucket-%s" % func.func_name,
        origin=q.name, status=JobStatus.QUEUED, timeout=ctx.timeout,
        result_ttl=0, ttl=ctx.ttl)

    for n in chunks(nargs, 100):
        job.created_at = datetime.utcnow()
        with connection.pipeline() as pipe:
            for s in n:
                argv[-1] = s
                job._id = unicode(uuid4())
                job.args = argv
                q.enqueue_job(job, pipeline=pipe)
            pipe.execute()


Instruction:
Explain what the following Python 3 code does
def page_strip(page, versioned):
    # page strip filtering should be conditional
    page.pop('ResponseMetadata', None)
    contents_key = versioned and 'Versions' or 'Contents'
    contents = page.get(contents_key, ())

    # aggressive size
    if versioned:
        keys = []
        for k in contents:
            if k['IsLatest']:
                keys.append((k['Key'], k['VersionId'], True))
            else:
                keys.append((k['Key'], k['VersionId']))
        return keys
    else:
        return [k['Key'] for k in contents]

    if not contents:
        return page

    # Depending on use case we may want these
    for k in contents:
        k.pop('Owner', None)
        k.pop('LastModified', None)
        k.pop('ETag', None)
        k.pop('StorageClass', None)
        k.pop('Size', None)

    return page

Response: Remove bits in content results to minimize memory utilization.


Instruction:
Write a Python 3 function for
scanning all buckets in an account and schedule processing

Response:
def process_account(account_info):
    """Scan all buckets in an account and schedule processing"""
    log = logging.getLogger('salactus.bucket-iterator')
    log.info("processing account %s", account_info)
    session = get_session(account_info)
    client = session.client('s3', config=s3config)
    buckets = client.list_buckets()['Buckets']

    connection.hset(
        'bucket-accounts', account_info['name'], json.dumps(account_info))

    for b in buckets:
        connection.hset(
            'bucket-ages', bucket_id(account_info, b['Name']),
            b['CreationDate'].isoformat())

    account_buckets = account_info.pop('buckets', None)
    buckets = [n['Name'] for n in buckets
               if not account_buckets or
               n['Name'] in account_buckets]
    account_not_buckets = account_info.pop('not-buckets', None)
    buckets = [n for n in buckets
               if not account_not_buckets or
               n not in account_not_buckets]
    log.info("processing %d buckets in account %s",
             len(buckets), account_info['name'])
    for bucket_set in chunks(buckets, 50):
        invoke(process_bucket_set, account_info, bucket_set)


Instruction:
Write a Python 3 script to
process a collection of buckets.

Response:
def process_bucket_set(account_info, buckets):
    """Process a collection of buckets.

    For each bucket fetch location, versioning and size and
    then kickoff processing strategy based on size.
    """
    region_clients = {}
    log = logging.getLogger('salactus.bucket-set')
    log.info("processing account %s", account_info)
    session = get_session(account_info)
    client = session.client('s3', config=s3config)

    for b in buckets:
        bid = bucket_id(account_info, b)
        with bucket_ops(bid):
            info = {'name': b}
            error = None

            try:
                location = client.get_bucket_location(
                    Bucket=b).get('LocationConstraint')
            except Exception as e:
                error = e
                location = None

            if location is None:
                region = "us-east-1"
            elif location == 'EU':
                region = "eu-west-1"
            else:
                region = location

            if (account_info.get('regions', ()) and
                    region not in account_info.get('regions', ())):
                continue

            info['region'] = region
            if region not in region_clients:
                region_clients.setdefault(region, {})
                region_clients[region]['s3'] = s3 = session.client(
                    's3', region_name=region, config=s3config)
                region_clients[region]['cloudwatch'] = cw = session.client(
                    'cloudwatch', region_name=region, config=s3config)
            else:
                s3 = region_clients[region]['s3']
                cw = region_clients[region]['cloudwatch']

            try:
                info['keycount'] = bucket_key_count(cw, info)
            except Exception:
                raise
            else:
                connection.hset('bucket-sizes', bid, info['keycount'])

            if error:
                raise error

            connection.hset('bucket-regions', bid, region)

            versioning = s3.get_bucket_versioning(Bucket=b)
            info['versioned'] = (
                versioning and versioning.get('Status', '')
                in ('Enabled', 'Suspended') or False)
            connection.hset('bucket-versions', bid, int(info['versioned']))

            log.info("processing bucket %s", info)
            connection.hset('bucket-starts', bid, time.time())
            dispatch_object_source(s3, account_info, bid, info)


Instruction:
Implement a Python 3 function for
selecting and dispatch an object source for a bucket.

Response:
def dispatch_object_source(client, account_info, bid, bucket_info):
    """Select and dispatch an object source for a bucket.

    Choices are bucket partition, inventory, or direct pagination.
    """

    if (account_info.get('inventory') and
        bucket_info['keycount'] >
            account_info['inventory'].get('bucket-size-threshold',
                                          DEFAULT_INVENTORY_BUCKET_SIZE_THRESHOLD)):
        inventory_info = get_bucket_inventory(
            client,
            bucket_info['name'],
            account_info['inventory'].get('id-selector', '*'))
        if inventory_info is not None:
            return invoke(
                process_bucket_inventory, bid,
                inventory_info['bucket'], inventory_info['prefix'])

    if bucket_info['keycount'] > PARTITION_BUCKET_SIZE_THRESHOLD:
        invoke(process_bucket_partitions, bid)
    else:
        invoke(process_bucket_iterator, bid)


Instruction:
Can you write a function in Python 3 where it
uses set of keys as selector for character superset of a keyset.

Response:
def get_keys_charset(keys, bid):
    """ Use set of keys as selector for character superset

    Note this isn't optimal, its probabilistic on the keyset char population.
    """
    # use the keys found to sample possible chars
    chars = set()
    for k in keys:
        chars.update(k[:4])
    remainder = chars

    # Normalize charsets for matching
    normalized = {}
    for n, sset in [
        ("p", set(string.punctuation)),
        ("w", set(string.whitespace))
    ]:
        m = chars.intersection(sset)
        if m:
            normalized[n] = m
            remainder = remainder.difference(sset)

    # Detect character sets
    charset = None
    for candidate in CharSet.charsets():
        if remainder.issubset(candidate):
            charset = candidate
            break

    if charset is None:
        raise ValueError(
            "Bucket: %s Failed charset ngram detection %r\n%s" % (
                bid, "".join(chars)), "\n".join(sorted(keys)))

    for n, sset in normalized.items():
        charset = charset.symmetric_difference(sset)

    return charset


Instruction:
Can you implement a function in Python 3 that
detects the partition strategy for a large bucket.

Response:
def detect_partition_strategy(bid, delimiters=('/', '-'), prefix=''):
    """Try to detect the best partitioning strategy for a large bucket

    Consider nested buckets with common prefixes, and flat buckets.
    """
    account, bucket = bid.split(":", 1)
    region = connection.hget('bucket-regions', bid)
    versioned = bool(int(connection.hget('bucket-versions', bid)))
    size = int(float(connection.hget('bucket-sizes', bid)))
    session = get_session(
        json.loads(connection.hget('bucket-accounts', account)))
    s3 = session.client('s3', region_name=region, config=s3config)

    (contents_key,
     contents_method,
     continue_tokens) = BUCKET_OBJ_DESC[versioned]

    with bucket_ops(bid, 'detect'):
        keys = set()
        for delimiter in delimiters:
            method = getattr(s3, contents_method, None)
            results = method(
                Bucket=bucket, Prefix=prefix, Delimiter=delimiter)
            prefixes = [p['Prefix'] for p in results.get('CommonPrefixes', [])]
            contents = results.get(contents_key, [])
            keys.update([k['Key'] for k in contents])
            # If we have common prefixes within limit thresholds go wide
            if (len(prefixes) > 0 and
                len(prefixes) < 1000 and
                    len(contents) < 1000):
                log.info(
                    "%s detected prefix delimiter:%s contents:%d prefixes:%d",
                    bid, delimiter, len(contents), len(prefixes))
                limit = prefix and 2 or 4
                return process_bucket_partitions(
                    bid, partition=delimiter,
                    strategy='p', prefix_set=prefixes, limit=limit)

    # Detect character sets
    charset = get_keys_charset(keys, bid)
    log.info("Detected charset %s for %s", charset, bid)

    # Determine the depth we need to keep total api calls below threshold
    scan_count = size / 1000.0
    for limit in range(1, 4):
        if math.pow(len(charset), limit) * 1000 > scan_count:
            break

    # Dispatch
    prefixes = ('',)
    prefixes = NGramPartition(
        charset, limit=limit).initialize_prefixes(prefixes)

    #
    random.shuffle(prefixes)

    # Pregen on ngram means we have many potentially useless prefixes
    # todo carry charset forward as param, and go incremental on prefix
    # ngram expansion
    connection.hincrby('bucket-partition', bid, len(prefixes))
    return bulk_invoke(
        process_bucket_iterator, [bid], prefixes)


Instruction:
How would you explain what the following Python 3 function does
def process_bucket_partitions(
        bid, prefix_set=('',), partition='/', strategy=None, limit=4):
    """Split up a bucket keyspace into smaller sets for parallel iteration.
    """
    if strategy is None:
        return detect_partition_strategy(bid)

    account, bucket = bid.split(':', 1)
    region = connection.hget('bucket-regions', bid)
    versioned = bool(int(connection.hget('bucket-versions', bid)))
    session = get_session(
        json.loads(connection.hget('bucket-accounts', account)))
    size = int(float(connection.hget('bucket-sizes', bid)))
    s3 = session.client('s3', region_name=region, config=s3config)

    strategy = get_partition_strategy(strategy)
    strategy.limit = limit
    strategy.partition = partition
    (contents_key,
     contents_method,
     continue_tokens) = BUCKET_OBJ_DESC[versioned]
    prefix_queue = strategy.initialize_prefixes(prefix_set)

    keyset = []
    log.info(
        "Process partition bid:%s strategy:%s delimiter:%s queue:%d limit:%d",
        bid, strategy.__class__.__name__[0], partition,
        len(prefix_queue), limit)

    def statm(prefix):
        return "keyset:%d queue:%d prefix:%s bucket:%s size:%d" % (
            len(keyset), len(prefix_queue), prefix, bid, size)

    while prefix_queue:
        connection.hincrby('bucket-partition', bid, 1)
        prefix = prefix_queue.pop()
        if strategy.is_depth_exceeded(prefix):
            log.info("Partition max depth reached, %s", statm(prefix))
            invoke(process_bucket_iterator, bid, prefix)
            continue
        method = getattr(s3, contents_method, None)
        results = method(Bucket=bucket, Prefix=prefix, Delimiter=partition)
        keyset.extend(results.get(contents_key, ()))

        # As we probe we find keys, process any found
        if len(keyset) > PARTITION_KEYSET_THRESHOLD:
            log.info("Partition, processing keyset %s", statm(prefix))
            page = page_strip({contents_key: keyset}, versioned)
            if page:
                invoke(process_keyset, bid, page)
            keyset = []

        strategy.find_partitions(prefix_queue, results)

        # Do we have more than 1k keys at this level, continue iteration
        continuation_params = {
            k: results[k] for k in continue_tokens if k in results}
        if continuation_params:
            bp = int(connection.hget('bucket-partition', bid))
            log.info("Partition has 1k keys, %s %s", statm(prefix), bp)
            if not prefix_queue and bp < 5:
                log.info("Recursive detection")
                return detect_partition_strategy(bid, prefix=prefix)

            invoke(process_bucket_iterator, bid, prefix, delimiter=partition,
                   **continuation_params)

        # If the queue get too deep, then go parallel
        if len(prefix_queue) > PARTITION_QUEUE_THRESHOLD:
            log.info("Partition add friends, %s", statm(prefix))
            for s_prefix_set in chunks(
                    prefix_queue[PARTITION_QUEUE_THRESHOLD - 1:],
                    PARTITION_QUEUE_THRESHOLD - 1):

                for s in list(s_prefix_set):
                    if strategy.is_depth_exceeded(prefix):
                        invoke(process_bucket_iterator, bid, s)
                        s_prefix_set.remove(s)

                if not s_prefix_set:
                    continue
                invoke(process_bucket_partitions,
                       bid,
                       prefix_set=s_prefix_set, partition=partition,
                       strategy=strategy, limit=limit)
            prefix_queue = prefix_queue[:PARTITION_QUEUE_THRESHOLD - 1]

    if keyset:
        page = page_strip({contents_key: keyset}, versioned)
        if page:
            invoke(process_keyset, bid, page)

Response: Process a set of partitions for a given bucket.


Instruction:
Can you generate the documentation for the following Python 3 function
def process_bucket_inventory(bid, inventory_bucket, inventory_prefix):
    log.info("Loading bucket %s keys from inventory s3://%s/%s",
             bid, inventory_bucket, inventory_prefix)
    account, bucket = bid.split(':', 1)
    region = connection.hget('bucket-regions', bid)
    versioned = bool(int(connection.hget('bucket-versions', bid)))
    session = boto3.Session()
    s3 = session.client('s3', region_name=region, config=s3config)

    # find any key visitors with inventory filtering
    account_info = json.loads(connection.hget('bucket-accounts', account))
    ifilters = [v.inventory_filter for v
                in get_key_visitors(account_info) if v.inventory_filter]

    with bucket_ops(bid, 'inventory'):
        page_iterator = load_bucket_inventory(
            s3, inventory_bucket, inventory_prefix, versioned, ifilters)
        if page_iterator is None:
            log.info("bucket:%s could not find inventory" % bid)
            # case: inventory configured but not delivered yet
            # action: dispatch to bucket partition (assumes 100k+ for inventory)
            # - todo consider max inventory age/staleness for usage
            return invoke(process_bucket_partitions, bid)
        connection.hset('buckets-inventory', bid, 1)
        for page in page_iterator:
            invoke(process_keyset, bid, page)

Response: Load last inventory dump and feed as key source.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def process_bucket_iterator(bid, prefix="", delimiter="", **continuation):
    log.info("Iterating keys bucket %s prefix %s delimiter %s",
             bid, prefix, delimiter)

    account, bucket = bid.split(':', 1)
    region = connection.hget('bucket-regions', bid)
    versioned = bool(int(connection.hget('bucket-versions', bid)))
    session = get_session(
        json.loads(connection.hget('bucket-accounts', account)))
    s3 = session.client('s3', region_name=region, config=s3config)

    (contents_key, contents_method, _) = BUCKET_OBJ_DESC[versioned]

    params = dict(Bucket=bucket)
    if prefix:
        params['Prefix'] = prefix
    if delimiter:
        params['Delimiter'] = delimiter
    if continuation:
        params.update({k[4:]: v for k, v in continuation.items()})
    paginator = s3.get_paginator(contents_method).paginate(**params)
    with bucket_ops(bid, 'page'):
        ptime = time.time()
        pcounter = 0
        for page in paginator:
            page = page_strip(page, versioned)
            pcounter += 1
            if page:
                invoke(process_keyset, bid, page)

            if pcounter % 10 == 0:
                with connection.pipeline() as p:
                    nptime = time.time()
                    p.hincrby('bucket-pages', bid, 1)
                    p.hincrby('bucket-pages-time', bid, int(nptime - ptime))
                    ptime = nptime
                    p.execute()

        if pcounter % 10:
            with connection.pipeline() as p:
                nptime = time.time()
                p.hincrby('bucket-pages', bid, 1)
                p.hincrby('bucket-pages-time', bid, int(nptime - ptime))
                p.execute()

Response: Process a single bucket iterator.


Instruction:
Given the following Python 3 function, write the documentation
def universal_retry(method, ResourceARNList, **kw):
    max_attempts = 6

    for idx, delay in enumerate(
            utils.backoff_delays(1.5, 2 ** 8, jitter=True)):
        response = method(ResourceARNList=ResourceARNList, **kw)
        failures = response.get('FailedResourcesMap', {})
        if not failures:
            return response

        errors = {}
        throttles = set()

        for f_arn in failures:
            error_code = failures[f_arn]['ErrorCode']
            if error_code == 'ThrottlingException':
                throttles.add(f_arn)
            elif error_code == 'ResourceNotFoundException':
                continue
            else:
                errors[f_arn] = error_code

        if errors:
            raise Exception("Resource Tag Errors %s" % (errors))

        if idx == max_attempts - 1:
            raise Exception("Resource Tag Throttled %s" % (", ".join(throttles)))

        time.sleep(delay)
        ResourceARNList = list(throttles)

Response: Universal retry support for resourcegroup tagging apis.


Instruction:
Explain what the following Python 3 code does
def coalesce_copy_user_tags(resource, copy_tags, user_tags):

    assert isinstance(copy_tags, bool) or isinstance(copy_tags, list)
    assert isinstance(user_tags, dict) or isinstance(user_tags, list)

    r_tags = resource.get('Tags', [])

    if isinstance(copy_tags, list):
        if '*' in copy_tags:
            copy_keys = set([t['Key'] for t in r_tags])
        else:
            copy_keys = set(copy_tags)

    if isinstance(copy_tags, bool):
        if copy_tags is True:
            copy_keys = set([t['Key'] for t in r_tags])
        else:
            copy_keys = set()

    if isinstance(user_tags, dict):
        user_tags = [{'Key': k, 'Value': v} for k, v in user_tags.items()]

    user_keys = set([t['Key'] for t in user_tags])
    tags_diff = list(copy_keys.difference(user_keys))
    resource_tags_to_copy = [t for t in r_tags if t['Key'] in tags_diff]
    user_tags.extend(resource_tags_to_copy)
    return user_tags

Response: Coalesce the tags from a resource and user supplied tags.


Instruction:
Create a Python 3 function for
processing a rename request.

Response:
def process_rename(self, client, tag_value, resource_set):
        """
        Move source tag value to destination tag value

        - Collect value from old tag
        - Delete old tag
        - Create new tag & assign stored value
        """
        self.log.info("Renaming tag on %s instances" % (len(resource_set)))
        old_key = self.data.get('old_key')
        new_key = self.data.get('new_key')

        # We have a preference to creating the new tag when possible first
        resource_ids = [r[self.id_key] for r in resource_set if len(
            r.get('Tags', [])) < self.tag_count_max]
        if resource_ids:
            self.create_tag(client, resource_ids, new_key, tag_value)

        self.delete_tag(
            client, [r[self.id_key] for r in resource_set], old_key, tag_value)

        # For resources with 50 tags, we need to delete first and then create.
        resource_ids = [r[self.id_key] for r in resource_set if len(
            r.get('Tags', [])) > self.tag_count_max - 1]
        if resource_ids:
            self.create_tag(client, resource_ids, new_key, tag_value)


Instruction:
Can you create a Python 3 function that
processes the transform tag value for the resource set

Response:
def process_transform(self, tag_value, resource_set):
        """
        Transform tag value

        - Collect value from tag
        - Transform Tag value
        - Assign new value for key
        """
        self.log.info("Transforming tag value on %s instances" % (
            len(resource_set)))
        key = self.data.get('key')

        c = utils.local_session(self.manager.session_factory).client('ec2')

        self.create_tag(
            c,
            [r[self.id_key] for r in resource_set if len(
                r.get('Tags', [])) < 50],
            key, tag_value)


Instruction:
Can you tell what is the following Python 3 function doing
def get_resource_tag_map(self, r_type, ids):
        manager = self.manager.get_resource_manager(r_type)
        r_id = manager.resource_type.id
        # TODO only fetch resource with the given ids.
        return {
            r[r_id]: {t['Key']: t['Value'] for t in r.get('Tags', [])}
            for r in manager.resources() if r[r_id] in ids
        }

Response: Returns a mapping of resource_id to tagkey - > tagvalue


Instruction:
Can you implement a function in Python 3 that
returns a mapping of launch configs for the given set of asgs

Response:
def get_launch_configs(self, asgs):
        """Return a mapping of launch configs for the given set of asgs"""
        config_names = set()
        for a in asgs:
            if 'LaunchConfigurationName' not in a:
                continue
            config_names.add(a['LaunchConfigurationName'])
        if not config_names:
            return {}
        lc_resources = self.manager.get_resource_manager('launch-config')
        if len(config_names) < 5:
            configs = lc_resources.get_resources(list(config_names))
        else:
            configs = lc_resources.resources()
        return {
            cfg['LaunchConfigurationName']: cfg for cfg in configs
            if cfg['LaunchConfigurationName'] in config_names}


Instruction:
Can you tell what is the following Python 3 function doing
def get_resources(self, ids, cache=True):
        if ids[0].startswith('arn:'):
            params = {'LoadBalancerArns': ids}
        else:
            params = {'Names': ids}
        return self.query.filter(self.manager, **params)

Response: Support server side filtering on arns or names


Instruction:
Can you generate the documentation for the following Python 3 function
def validate(config):
    with open(config) as fh:
        data = utils.yaml_load(fh.read())
        jsonschema.validate(data, CONFIG_SCHEMA)

Response: Validate a configuration file.


Instruction:
Write a Python 3 script for
running across a set of accounts and buckets.

Response:
def run(config, tag, bucket, account, not_bucket, not_account, debug, region):
    """Run across a set of accounts and buckets."""
    logging.basicConfig(
        level=logging.INFO,
        format="%(asctime)s: %(name)s:%(levelname)s %(message)s")
    logging.getLogger('botocore').setLevel(level=logging.WARNING)

    if debug:
        def invoke(f, *args, **kw):
            # if f.func_name == 'process_keyset':
            #    key_count = len(args[-1])
            #    print("debug skip keyset %d" % key_count)
            #    return
            return f(*args, **kw)
        worker.invoke = invoke

    with open(config) as fh:
        data = utils.yaml_load(fh.read())
        for account_info in data.get('accounts', ()):
            if tag and tag not in account_info.get('tags', ()):
                continue
            if account and account_info['name'] not in account:
                continue
            if not_account and account_info['name'] in not_account:
                continue
            if 'inventory' in data and 'inventory' not in account_info:
                account_info['inventory'] = data['inventory']
            if 'visitors' in data and 'visitors' not in account_info:
                account_info['visitors'] = data['visitors']
            if 'object-reporting' in data and 'object-reporting' not in account_info:
                account_info['object-reporting'] = data['object-reporting']
                account_info['object-reporting'][
                    'record-prefix'] = datetime.utcnow().strftime('%Y/%m/%d')
            if bucket:
                account_info['buckets'] = bucket
            if not_bucket:
                account_info['not-buckets'] = not_bucket
            if region:
                account_info['regions'] = region

            try:
                worker.invoke(worker.process_account, account_info)
            except Exception:
                if not debug:
                    raise
                import pdb, traceback, sys
                traceback.print_exc()
                pdb.post_mortem(sys.exc_info()[-1])
                raise


Instruction:
Write a Python 3 script for
deleting all persistent cluster state.

Response:
def reset(c7n_async=None):
    """Delete all persistent cluster state.
    """
    click.echo('Delete db? Are you Sure? [yn] ', nl=False)
    c = click.getchar()
    click.echo()
    if c == 'y':
        click.echo('Wiping database')
        worker.connection.flushdb()
    elif c == 'n':
        click.echo('Abort!')
    else:
        click.echo('Invalid input :(')


Instruction:
Write a Python 3 script to
show information on salactus workers.

Response:
def workers():
    """Show information on salactus workers. (slow)"""
    counter = Counter()
    for w in Worker.all(connection=worker.connection):
        for q in w.queues:
            counter[q.name] += 1
    import pprint
    pprint.pprint(dict(counter))


Instruction:
Write a Python 3 function for
reporting on stats by account

Response:
def accounts(dbpath, output, format, account,
             config=None, tag=None, tagprefix=None, region=(),
             not_region=(), not_bucket=None):
    """Report on stats by account"""
    d = db.db(dbpath)
    accounts = d.accounts()
    formatter = (
        format == 'csv' and format_accounts_csv or format_accounts_plain)

    if region:
        for a in accounts:
            a.buckets = [b for b in a.buckets if b.region in region]
        accounts = [a for a in accounts if a.bucket_count]

    if not_region:
        for a in accounts:
            a.buckets = [b for b in a.buckets if b.region not in not_region]
        accounts = [a for a in accounts if a.bucket_count]

    if not_bucket:
        for a in accounts:
            a.buckets = [b for b in a.buckets if b.name not in not_bucket]
    if config and tagprefix:
        account_map = {account.name: account for account in accounts}

        with open(config) as fh:
            account_data = json.load(fh).get('accounts')
        tag_groups = {}
        for a in account_data:
            if tag is not None and tag not in a['tags']:
                continue

            for t in a['tags']:
                if t.startswith(tagprefix):
                    tvalue = t[len(tagprefix):]
                    if not tvalue:
                        continue
                    if tvalue not in tag_groups:
                        tag_groups[tvalue] = db.Account(tvalue, [])
                    account_results = account_map.get(a['name'])
                    if not account_results:
                        print("missing %s" % a['name'])
                        continue
                    tag_groups[tvalue].buckets.extend(
                        account_map[a['name']].buckets)
        accounts = tag_groups.values()

    formatter(accounts, output)


Instruction:
How would you code a function in Python 3 to
report on stats by bucket

Response:
def buckets(bucket=None, account=None, matched=False, kdenied=False,
            errors=False, dbpath=None, size=None, denied=False,
            format=None, incomplete=False, oversize=False, region=(),
            not_region=(), inventory=None, output=None, config=None, sort=None,
            tagprefix=None, not_bucket=None):
    """Report on stats by bucket"""

    d = db.db(dbpath)

    if tagprefix and not config:
        raise ValueError(
            "account tag value inclusion requires account config file")

    if config and tagprefix:
        with open(config) as fh:
            data = json.load(fh).get('accounts')
            account_data = {}
            for a in data:
                for t in a['tags']:
                    if t.startswith(tagprefix):
                        account_data[a['name']] = t[len(tagprefix):]

    buckets = []
    for b in sorted(d.buckets(account),
                    key=operator.attrgetter('bucket_id')):
        if bucket and b.name not in bucket:
            continue
        if not_bucket and b.name in not_bucket:
            continue
        if matched and not b.matched:
            continue
        if kdenied and not b.keys_denied:
            continue
        if errors and not b.error_count:
            continue
        if size and b.size < size:
            continue
        if inventory and not b.using_inventory:
            continue
        if denied and not b.denied:
            continue
        if oversize and b.scanned <= b.size:
            continue
        if incomplete and b.percent_scanned >= incomplete:
            continue
        if region and b.region not in region:
            continue
        if not_region and b.region in not_region:
            continue
        if tagprefix:
            setattr(b, tagprefix[:-1], account_data[b.account])
        buckets.append(b)

    if sort:
        key = operator.attrgetter(sort)
        buckets = list(reversed(sorted(buckets, key=key)))
    formatter = format == 'csv' and format_csv or format_plain
    keys = tagprefix and (tagprefix[:-1],) or ()
    formatter(buckets, output, keys=keys)


Instruction:
Here you have a function in Python 3, explain what it does
def watch(limit):
    period = 5.0
    prev = db.db()
    prev_totals = None

    while True:
        click.clear()
        time.sleep(period)
        cur = db.db()
        cur.data['gkrate'] = {}
        progress = []
        prev_buckets = {b.bucket_id: b for b in prev.buckets()}

        totals = {'scanned': 0, 'krate': 0, 'lrate': 0, 'bucket_id': 'totals'}

        for b in cur.buckets():
            if not b.scanned:
                continue

            totals['scanned'] += b.scanned
            totals['krate'] += b.krate
            totals['lrate'] += b.lrate

            if b.bucket_id not in prev_buckets:
                b.data['gkrate'][b.bucket_id] = b.scanned / period
            elif b.scanned == prev_buckets[b.bucket_id].scanned:
                continue
            else:
                b.data['gkrate'][b.bucket_id] = (
                    b.scanned - prev_buckets[b.bucket_id].scanned) / period
            progress.append(b)

        if prev_totals is None:
            totals['gkrate'] = '...'
        else:
            totals['gkrate'] = (totals['scanned'] - prev_totals['scanned']) / period
        prev = cur
        prev_totals = totals

        progress = sorted(progress, key=lambda x: x.gkrate, reverse=True)

        if limit:
            progress = progress[:limit]

        progress.insert(0, Bag(totals))
        format_plain(
            progress, None,
            explicit_only=True,
            keys=['bucket_id', 'scanned', 'gkrate', 'lrate', 'krate'])

Response: watch scan rates across the cluster


Instruction:
How would you code a function in Python 3 to
discover the partitions on a bucket via introspection. For large buckets which lack s3 inventories, salactus will attempt to process objects in parallel on the bucket by breaking the bucket into a separate keyspace partitions. It does this with a heurestic that attempts to sample the keyspace and determine appropriate subparts. This command provides additional visibility into the partitioning of a bucket by showing how salactus would partition a given bucket.

Response:
def inspect_partitions(bucket):
    """Discover the partitions on a bucket via introspection.

    For large buckets which lack s3 inventories, salactus will attempt
    to process objects in parallel on the bucket by breaking the bucket
    into a separate keyspace partitions. It does this with a heurestic
    that attempts to sample the keyspace and determine appropriate subparts.

    This command provides additional visibility into the partitioning of
    a bucket by showing how salactus would partition a given bucket.
    """

    logging.basicConfig(
        level=logging.INFO,
        format="%(asctime)s: %(name)s:%(levelname)s %(message)s")
    logging.getLogger('botocore').setLevel(level=logging.WARNING)

    state = db.db()
    # add db.bucket accessor
    found = None
    for b in state.buckets():
        if b.name == bucket:
            found = b
            break
    if not found:
        click.echo("no bucket named: %s" % bucket)
        return

    keyset = []
    partitions = []

    def process_keyset(bid, page):
        keyset.append(len(page))

    def process_bucket_iterator(bid, prefix, delimiter="", **continuation):
        partitions.append(prefix)

    # synchronous execution
    def invoke(f, *args, **kw):
        return f(*args, **kw)

    # unleash the monkies ;-)
    worker.connection.hincrby = lambda x, y, z: True
    worker.invoke = invoke
    worker.process_keyset = process_keyset
    worker.process_bucket_iterator = process_bucket_iterator

    # kick it off
    worker.process_bucket_partitions(b.bucket_id)

    keys_scanned = sum(keyset)
    click.echo(
        "Found %d partitions %s keys scanned during partitioning" % (
            len(partitions), keys_scanned))
    click.echo("\n".join(partitions))


Instruction:
Can you generate the documentation for the following Python 3 function
def inspect_bucket(bucket):
    state = db.db()
    found = None
    for b in state.buckets():
        if b.name == bucket:
            found = b
    if not found:
        click.echo("no bucket named: %s" % bucket)
        return

    click.echo("Bucket: %s" % found.name)
    click.echo("Account: %s" % found.account)
    click.echo("Region: %s" % found.region)
    click.echo("Created: %s" % found.created)
    click.echo("Size: %s" % found.size)
    click.echo("Inventory: %s" % found.inventory)
    click.echo("Partitions: %s" % found.partitions)
    click.echo("Scanned: %0.2f%%" % found.percent_scanned)
    click.echo("")
    click.echo("Errors")

    click.echo("Denied: %s" % found.keys_denied)
    click.echo("BErrors: %s" % found.error_count)
    click.echo("KErrors: %s" % found.data['keys-error'].get(found.bucket_id, 0))
    click.echo("Throttle: %s" % found.data['keys-throttled'].get(found.bucket_id, 0))
    click.echo("Missing: %s" % found.data['keys-missing'].get(found.bucket_id, 0))
    click.echo("Session: %s" % found.data['keys-sesserr'].get(found.bucket_id, 0))
    click.echo("Connection: %s" % found.data['keys-connerr'].get(found.bucket_id, 0))
    click.echo("Endpoint: %s" % found.data['keys-enderr'].get(found.bucket_id, 0))

Response: Show all information known on a bucket.


Instruction:
How would you implement a function in Python 3 that
shows contents of a queue.

Response:
def inspect_queue(queue, state, limit, bucket):
    """Show contents of a queue."""
    if not HAVE_BIN_LIBS:
        click.echo("missing required binary libs (lz4, msgpack)")
        return

    conn = worker.connection

    def job_row(j):
        if isinstance(j.args[0], basestring):
            account, bucket = j.args[0].split(':', 1)
        elif isinstance(j.args[0], dict):
            account, bucket = j.args[0]['name'], "set %d" % len(j.args[1])

        row = {
            'account': account,
            'bucket': bucket,
            # 'region': j.args[1]['region'],
            # 'size': j.args[1]['keycount'],
            'ttl': j.ttl,
            'enqueued': j.enqueued_at,
            'rtt': j.result_ttl,
            'timeout': j.timeout}

        if queue != "bucket-keyset-scan":
            row['args'] = j.args[2:]
        if state in ('running', 'failed', 'finished'):
            row['started'] = j.started_at
        if state in ('finished', 'failed'):
            row['ended'] = j.ended_at
        return row

    if state == 'running':
        registry_class = StartedJobRegistry
    elif state == 'pending':
        registry_class = Queue
    elif state == 'failed':
        registry_class = FailedQueue
    elif state == 'finished':
        registry_class = FinishedJobRegistry
    else:
        raise ValueError("invalid state: %s" % state)

    registry = registry_class(queue, connection=conn)
    records = []
    for jid in registry.get_job_ids():
        j = Job.fetch(jid, conn)
        if bucket:
            if j.args[1]['name'] != bucket:
                continue
        records.append(job_row(j))
        if len(records) == limit:
            break
    if records:
        click.echo(
            tabulate.tabulate(
                records,
                "keys",
                tablefmt='simple'))
    else:
        click.echo("no queue items found")


Instruction:
Can you implement a function in Python 3 that
reports on progress by queues.

Response:
def queues():
    """Report on progress by queues."""
    conn = worker.connection
    failure_q = None

    def _repr(q):
        return "running:%d pending:%d finished:%d" % (
            StartedJobRegistry(q.name, conn).count,
            q.count,
            FinishedJobRegistry(q.name, conn).count)
    for q in Queue.all(conn):
        if q.name == 'failed':
            failure_q = q
            continue
        click.echo("%s %s" % (q.name, _repr(q)))
    if failure_q:
        click.echo(
            click.style(failure_q.name, fg='red') + ' %s' % _repr(failure_q))


Instruction:
Can you create a Python 3 function that
shows any unexpected failures

Response:
def failures():
    """Show any unexpected failures"""
    if not HAVE_BIN_LIBS:
        click.echo("missing required binary libs (lz4, msgpack)")
        return

    q = Queue('failed', connection=worker.connection)
    for i in q.get_job_ids():
        j = q.job_class.fetch(i, connection=q.connection)
        click.echo("%s on %s" % (j.func_name, j.origin))
        if not j.func_name.endswith('process_keyset'):
            click.echo("params %s %s" % (j._args, j._kwargs))
        click.echo(j.exc_info)


Instruction:
Write a Python 3 function for
generating a c7n - org subscriptions config file

Response:
def main(output):
    """
    Generate a c7n-org subscriptions config file
    """

    client = SubscriptionClient(Session().get_credentials())
    subs = [sub.serialize(True) for sub in client.subscriptions.list()]
    results = []
    for sub in subs:
        sub_info = {
            'subscription_id': sub['subscriptionId'],
            'name': sub['displayName']
        }
        results.append(sub_info)

    print(
        yaml.safe_dump(
            {'subscriptions': results},
            default_flow_style=False),
        file=output)


Instruction:
Can you generate the documentation for the following Python 3 function
def custodian_azure_send_override(self, request, headers=None, content=None, **kwargs):
    retries = 0
    max_retries = 3
    while retries < max_retries:
        response = self.orig_send(request, headers, content, **kwargs)

        send_logger.debug(response.status_code)
        for k, v in response.headers.items():
            if k.startswith('x-ms-ratelimit'):
                send_logger.debug(k + ':' + v)

        # Retry codes from urllib3/util/retry.py
        if response.status_code in [413, 429, 503]:
            retry_after = None
            for k in response.headers.keys():
                if StringUtils.equal('retry-after', k):
                    retry_after = int(response.headers[k])

            if retry_after is not None and retry_after < constants.DEFAULT_MAX_RETRY_AFTER:
                send_logger.warning('Received retriable error code %i. Retry-After: %i'
                                    % (response.status_code, retry_after))
                time.sleep(retry_after)
                retries += 1
            else:
                send_logger.error("Received throttling error, retry time is %i"
                                  "(retry only if < %i seconds)."
                                  % (retry_after, constants.DEFAULT_MAX_RETRY_AFTER))
                break
        else:
            break
    return response

Response: Override send function to implement retries & log headers


Instruction:
Given the following Python 3 function, write the documentation
def get_principal_dictionary(graph_client, object_ids, raise_on_graph_call_error=False):
        if not object_ids:
            return {}

        object_params = GetObjectsParameters(
            include_directory_object_references=True,
            object_ids=object_ids)

        principal_dics = {object_id: DirectoryObject() for object_id in object_ids}

        aad_objects = graph_client.objects.get_objects_by_object_ids(object_params)
        try:
            for aad_object in aad_objects:
                principal_dics[aad_object.object_id] = aad_object

        except CloudError as e:
            if e.status_code in [403, 401]:
                GraphHelper.log.warning(
                    'Credentials not authorized for access to read from Microsoft Graph. \n '
                    'Can not query on principalName, displayName, or aadType. \n')
            else:
                GraphHelper.log.error(
                    'Exception in call to Microsoft Graph. \n '
                    'Can not query on principalName, displayName, or aadType. \n'
                    'Error: {0}'.format(e))

            if raise_on_graph_call_error:
                raise

        return principal_dics

Response: Retrieves the principal dictionary for the given object ids.


Instruction:
How would you implement a function in Python 3 that
attempts to resolve a principal name.

Response:
def get_principal_name(graph_object):
        """Attempts to resolve a principal name.
        :param graph_object: the Azure AD Graph Object
        :return: The resolved value or an empty string if unsuccessful.
        """
        if hasattr(graph_object, 'user_principal_name'):
            return graph_object.user_principal_name
        elif hasattr(graph_object, 'service_principal_names'):
            return graph_object.service_principal_names[0]
        elif hasattr(graph_object, 'display_name'):
            return graph_object.display_name
        return ''


Instruction:
Create a Python 3 function for
returning a tuple with start and end ports

Response:
def _get_port_range(range_str):
        """ Given a string with a port or port range: '80', '80-120'
            Returns tuple with range start and end ports: (80, 80), (80, 120)
        """
        if range_str == '*':
            return PortsRangeHelper.PortsRange(start=0, end=65535)

        s = range_str.split('-')
        if len(s) == 2:
            return PortsRangeHelper.PortsRange(start=int(s[0]), end=int(s[1]))

        return PortsRangeHelper.PortsRange(start=int(s[0]), end=int(s[0]))


Instruction:
How would you explain what the following Python 3 function does
def _get_rule_port_ranges(rule):
        properties = rule['properties']
        if 'destinationPortRange' in properties:
            return [PortsRangeHelper._get_port_range(properties['destinationPortRange'])]
        else:
            return [PortsRangeHelper._get_port_range(r)
                    for r in properties['destinationPortRanges']]

Response:
Extracts the ports ranges from the NSG rule object
            Returns an array of PortsRange tuples


Instruction:
Can you generate the documentation for the following Python 3 function
def _port_ranges_to_set(ranges):
        return set([i for r in ranges for i in range(r.start, r.end + 1)])

Response: Converts an array of port ranges to a set of integers


Instruction:
Write a Python 3 script for
validating that the string has proper port numbers

Response:
def validate_ports_string(ports):
        """ Validate that provided string has proper port numbers:
            1. port number < 65535
            2. range start < range end
        """
        pattern = re.compile('^\\d+(-\\d+)?(,\\d+(-\\d+)?)*$')
        if pattern.match(ports) is None:
            return False

        ranges = PortsRangeHelper._get_string_port_ranges(ports)
        for r in ranges:
            if r.start > r.end or r.start > 65535 or r.end > 65535:
                return False
        return True


Instruction:
How would you implement a function in Python 3 that
transforms a list of port numbers to the list of strings with port ranges Example : 10 12 - 13 14 - 15

Response:
def get_ports_strings_from_list(data):
        """ Transform a list of port numbers to the list of strings with port ranges
            Example: [10, 12, 13, 14, 15] -> ['10', '12-15']
        """
        if len(data) == 0:
            return []

        # Transform diff_ports list to the ranges list
        first = 0
        result = []
        for it in range(1, len(data)):
            if data[first] == data[it] - (it - first):
                continue
            result.append(PortsRangeHelper.PortsRange(start=data[first], end=data[it - 1]))
            first = it

        # Update tuples with strings, representing ranges
        result.append(PortsRangeHelper.PortsRange(start=data[first], end=data[-1]))
        result = [str(x.start) if x.start == x.end else "%i-%i" % (x.start, x.end) for x in result]
        return result


Instruction:
Here you have a function in Python 3, explain what it does
def build_ports_dict(nsg, direction_key, ip_protocol):
        rules = nsg['properties']['securityRules']
        rules = sorted(rules, key=lambda k: k['properties']['priority'])
        ports = {}

        for rule in rules:
            # Skip rules with different direction
            if not StringUtils.equal(direction_key, rule['properties']['direction']):
                continue

            # Check the protocol: possible values are 'TCP', 'UDP', '*' (both)
            # Skip only if rule and ip_protocol are 'TCP'/'UDP' pair.
            protocol = rule['properties']['protocol']
            if not StringUtils.equal(protocol, "*") and \
               not StringUtils.equal(ip_protocol, "*") and \
               not StringUtils.equal(protocol, ip_protocol):
                continue

            IsAllowed = StringUtils.equal(rule['properties']['access'], 'allow')
            ports_set = PortsRangeHelper.get_ports_set_from_rule(rule)

            for p in ports_set:
                if p not in ports:
                    ports[p] = IsAllowed

        return ports

Response: Builds a dictionary of ports based on the provided Network Security Group object direction and protocol.


Instruction:
Can you generate the documentation for the following Python 3 function
def get_name(self, r):
        namer = ResourceNameAdapters[self.manager.resource_type.service]
        return namer(r)

Response: Given an arbitrary resource attempt to resolve back to a qualified name.


Instruction:
Given the following Python 3 function, write the documentation
def update_headers(src_tree):
    print("src tree", src_tree)
    for root, dirs, files in os.walk(src_tree):
        py_files = fnmatch.filter(files, "*.py")
        for f in py_files:
            print("checking", f)
            p = os.path.join(root, f)
            with open(p) as fh:
                contents = fh.read()
            if suffix in contents:
                continue
            print("Adding license header to %s" % p)
            with open(p, 'w') as fh:
                fh.write(
                    '%s%s%s' % (header, suffix, contents))

Response: Update the license headers for all the modules in src_tree.


Instruction:
Can you create a Python 3 function that
shows extant locks and unlocks.

Response:
def list_locks(account_id, resource_type=None, resource_id=None):
    """Show extant locks and unlocks.
    """
    locks = Client(BASE_URL, account_id).list_locks().json()

    for r in locks:
        if 'LockDate' in r:
            r['LockDate'] = datetime.fromtimestamp(r['LockDate'])
        if 'RevisionDate' in r:
            r['RevisionDate'] = datetime.fromtimestamp(r['RevisionDate'])

    print(tabulate.tabulate(
        locks,
        headers="keys",
        tablefmt='fancy_grid'))


Instruction:
Create a Python 3 function for
showing extant locks status

Response:
def lock_status(account_id, resource_id, parent_id):
    """Show extant locks' status
    """
    return output(
        Client(BASE_URL, account_id).lock_status(resource_id, parent_id))


Instruction:
Write a Python 3 script for
locking a resource in a given account

Response:
def lock(account_id, resource_id, region):
    """Lock a resource
    """
    return output(
        Client(BASE_URL, account_id).lock(resource_id, region))


Instruction:
Given the following Python 3 function, write the documentation
def list_locks(self, account_id=None):
        account_id = self.get_account_id(account_id)
        return self.http.get(
            "%s/%s/locks" % (self.endpoint, account_id),
            auth=self.get_api_auth())

Response: Get extant locks for the given account.


Instruction:
Implement a function in Python 3 to
get the lock status for a given resource.

Response:
def lock_status(self, resource_id, parent_id=None, account_id=None):
        """Get the lock status for a given resource.

        for security groups, parent id is their vpc.
        """
        account_id = self.get_account_id(account_id)
        params = parent_id and {'parent_id': parent_id} or None
        return self.http.get(
            "%s/%s/locks/%s" % (self.endpoint, account_id, resource_id),
            params=params, auth=self.get_api_auth())


Instruction:
Write a Python 3 script for
locking a given resource in a given region.

Response:
def lock(self, resource_id, region, account_id=None):
        """Lock a given resource
        """
        account_id = self.get_account_id(account_id)
        return self.http.post(
            "%s/%s/locks/%s/lock" % (self.endpoint, account_id, resource_id),
            json={'region': region},
            auth=self.get_api_auth())


Instruction:
Write a Python 3 script for
reporting on a cross account policy execution.

Response:
def report(config, output, use, output_dir, accounts,
           field, no_default_fields, tags, region, debug, verbose,
           policy, policy_tags, format, resource, cache_path):
    """report on a cross account policy execution."""
    accounts_config, custodian_config, executor = init(
        config, use, debug, verbose, accounts, tags, policy,
        resource=resource, policy_tags=policy_tags)

    resource_types = set()
    for p in custodian_config.get('policies'):
        resource_types.add(p['resource'])
    if len(resource_types) > 1:
        raise ValueError("can only report on one resource type at a time")
    elif not len(custodian_config['policies']) > 0:
        raise ValueError("no matching policies found")

    records = []
    with executor(max_workers=WORKER_COUNT) as w:
        futures = {}
        for a in accounts_config.get('accounts', ()):
            for r in resolve_regions(region or a.get('regions', ())):
                futures[w.submit(
                    report_account,
                    a, r,
                    custodian_config,
                    output_dir,
                    cache_path,
                    debug)] = (a, r)

        for f in as_completed(futures):
            a, r = futures[f]
            if f.exception():
                if debug:
                    raise
                log.warning(
                    "Error running policy in %s @ %s exception: %s",
                    a['name'], r, f.exception())
            records.extend(f.result())

    log.debug(
        "Found %d records across %d accounts and %d policies",
        len(records), len(accounts_config['accounts']),
        len(custodian_config['policies']))

    if format == 'json':
        dumps(records, output, indent=2)
        return

    prefix_fields = OrderedDict(
        (('Account', 'account'), ('Region', 'region'), ('Policy', 'policy')))
    config = Config.empty()
    factory = resource_registry.get(list(resource_types)[0])

    formatter = Formatter(
        factory.resource_type,
        extra_fields=field,
        include_default_fields=not(no_default_fields),
        include_region=False,
        include_policy=False,
        fields=prefix_fields)

    rows = formatter.to_csv(records, unique=False)
    writer = UnicodeWriter(output, formatter.headers())
    writer.writerow(formatter.headers())
    writer.writerows(rows)