Instruction:
Create a Python 3 function for
deleting images not in use or installed via image_present .. warning:: Only images installed via image_present states that are included via the top file will be detected.

Response:
def image_vacuum(name):
    '''
    Delete images not in use or installed via image_present

    .. warning::

        Only image_present states that are included via the
        top file will be detected.
    '''
    name = name.lower()
    ret = {'name': name,
           'changes': {},
           'result': None,
           'comment': ''}

    # list of images to keep
    images = []

    # retrieve image_present state data for host
    for state in __salt__['state.show_lowstate']():
        # don't throw exceptions when not highstate run
        if 'state' not in state:
            continue

        # skip if not from this state module
        if state['state'] != __virtualname__:
            continue
        # skip if not image_present
        if state['fun'] not in ['image_present']:
            continue
        # keep images installed via image_present
        if 'name' in state:
            if _is_uuid(state['name']):
                images.append(state['name'])
            elif _is_docker_uuid(state['name']):
                state['name'] = __salt__['imgadm.docker_to_uuid'](state['name'])
                if not state['name']:
                    continue
                images.append(state['name'])

    # retrieve images in use by vms
    for image_uuid in __salt__['vmadm.list'](order='image_uuid'):
        if image_uuid not in images:
            images.append(image_uuid)

    # purge unused images
    ret['result'] = True
    for image_uuid in __salt__['imgadm.list']():
        if image_uuid in images:
            continue

        image = __salt__['imgadm.get'](image_uuid)
        if image['manifest']['name'] == 'docker-layer':
            # NOTE: docker images are made of multiple layers, loop over them
            while image:
                image_uuid = image['manifest']['uuid']
                if image_uuid in __salt__['imgadm.delete'](image_uuid):
                    ret['changes'][image_uuid] = None
                else:
                    ret['result'] = False
                    ret['comment'] = 'failed to delete images'
                if 'origin' in image['manifest']:
                    image = __salt__['imgadm.get'](image['manifest']['origin'])
                else:
                    image = None
        else:
            # NOTE: normal images can just be delete
            if image_uuid in __salt__['imgadm.delete'](image_uuid):
                ret['changes'][image_uuid] = None
            else:
                ret['result'] = False
                ret['comment'] = 'failed to delete images'

    if ret['result'] and not ret['changes']:
        ret['comment'] = 'no images deleted'
    elif ret['result'] and ret['changes']:
        ret['comment'] = 'images deleted'

    return ret


Instruction:
Given the following Python 3 function, write the documentation
def vm_present(name, vmconfig, config=None):
    '''
    Ensure vm is present on the computenode

    name : string
        hostname of vm
    vmconfig : dict
        options to set for the vm
    config : dict
        fine grain control over vm_present

    .. note::

        The following configuration properties can be toggled in the config parameter.
          - kvm_reboot (true)                - reboots of kvm zones if needed for a config update
          - auto_import (false)              - automatic importing of missing images
          - auto_lx_vars (true)              - copy kernel_version and docker:* variables from image
          - reprovision (false)              - reprovision on image_uuid changes
          - enforce_tags (true)              - false = add tags only, true =  add, update, and remove tags
          - enforce_routes (true)            - false = add tags only, true =  add, update, and remove routes
          - enforce_internal_metadata (true) - false = add metadata only, true =  add, update, and remove metadata
          - enforce_customer_metadata (true) - false = add metadata only, true =  add, update, and remove metadata

    .. note::

        State ID is used as hostname. Hostnames must be unique.

    .. note::

        If hostname is provided in vmconfig this will take president over the State ID.
        This allows multiple states to be applied to the same vm.

    .. note::

        The following instances should have a unique ID.
          - nic : mac
          - filesystem: target
          - disk : path or diskN for zvols

        e.g. disk0 will be the first disk added, disk1 the 2nd,...

    .. versionchanged:: 2019.2.0

        Added support for docker image uuids, added auto_lx_vars configuration, documented some missing configuration options.

    '''
    name = name.lower()
    ret = {'name': name,
           'changes': {},
           'result': None,
           'comment': ''}

    # config defaults
    state_config = config if config else {}
    config = {
        'kvm_reboot': True,
        'auto_import': False,
        'auto_lx_vars': True,
        'reprovision': False,
        'enforce_tags': True,
        'enforce_routes': True,
        'enforce_internal_metadata': True,
        'enforce_customer_metadata': True,
    }
    config.update(state_config)
    log.debug('smartos.vm_present::%s::config - %s', name, config)

    # map special vmconfig parameters
    #  collections have set/remove handlers
    #  instances have add/update/remove handlers and a unique id
    vmconfig_type = {
        'collection': [
            'tags',
            'customer_metadata',
            'internal_metadata',
            'routes'
        ],
        'instance': {
            'nics': 'mac',
            'disks': 'path',
            'filesystems': 'target'
        },
        'create_only': [
            'filesystems'
        ]
    }
    vmconfig_docker_keep = [
        'docker:id',
        'docker:restartcount',
    ]
    vmconfig_docker_array = [
        'docker:env',
        'docker:cmd',
        'docker:entrypoint',
    ]

    # parse vmconfig
    vmconfig = _parse_vmconfig(vmconfig, vmconfig_type['instance'])
    log.debug('smartos.vm_present::%s::vmconfig - %s', name, vmconfig)

    # set hostname if needed
    if 'hostname' not in vmconfig:
        vmconfig['hostname'] = name

    # prepare image_uuid
    if 'image_uuid' in vmconfig:
        # NOTE: lookup uuid from docker uuid (normal uuid's are passed throuhg unmodified)
        #       we must do this again if we end up importing a missing image later!
        docker_uuid = __salt__['imgadm.docker_to_uuid'](vmconfig['image_uuid'])
        vmconfig['image_uuid'] = docker_uuid if docker_uuid else vmconfig['image_uuid']

        # NOTE: import image (if missing and allowed)
        if vmconfig['image_uuid'] not in __salt__['imgadm.list']():
            if config['auto_import']:
                if not __opts__['test']:
                    res = __salt__['imgadm.import'](vmconfig['image_uuid'])
                    vmconfig['image_uuid'] = __salt__['imgadm.docker_to_uuid'](vmconfig['image_uuid'])
                    if vmconfig['image_uuid'] not in res:
                        ret['result'] = False
                        ret['comment'] = 'failed to import image {0}'.format(vmconfig['image_uuid'])
            else:
                ret['result'] = False
                ret['comment'] = 'image {0} not installed'.format(vmconfig['image_uuid'])

    # prepare disk.*.image_uuid
    for disk in vmconfig['disks'] if 'disks' in vmconfig else []:
        if 'image_uuid' in disk and disk['image_uuid'] not in __salt__['imgadm.list']():
            if config['auto_import']:
                if not __opts__['test']:
                    res = __salt__['imgadm.import'](disk['image_uuid'])
                    if disk['image_uuid'] not in res:
                        ret['result'] = False
                        ret['comment'] = 'failed to import image {0}'.format(disk['image_uuid'])
            else:
                ret['result'] = False
                ret['comment'] = 'image {0} not installed'.format(disk['image_uuid'])

    # docker json-array handling
    if 'internal_metadata' in vmconfig:
        for var in vmconfig_docker_array:
            if var not in vmconfig['internal_metadata']:
                continue
            if isinstance(vmconfig['internal_metadata'][var], list):
                vmconfig['internal_metadata'][var] = json.dumps(
                    vmconfig['internal_metadata'][var]
                )

    # copy lx variables
    if vmconfig['brand'] == 'lx' and config['auto_lx_vars']:
        # NOTE: we can only copy the lx vars after the image has bene imported
        vmconfig = _copy_lx_vars(vmconfig)

    # quick abort if things look wrong
    # NOTE: use explicit check for false, otherwise None also matches!
    if ret['result'] is False:
        return ret

    # check if vm exists
    if vmconfig['hostname'] in __salt__['vmadm.list'](order='hostname'):
        # update vm
        ret['result'] = True

        # expand vmconfig
        vmconfig = {
            'state': vmconfig,
            'current': __salt__['vmadm.get'](vmconfig['hostname'], key='hostname'),
            'changed': {},
            'reprovision_uuid': None
        }

        # prepare reprovision
        if 'image_uuid' in vmconfig['state']:
            vmconfig['reprovision_uuid'] = vmconfig['state']['image_uuid']
            vmconfig['state']['image_uuid'] = vmconfig['current']['image_uuid']

        # disks need some special care
        if 'disks' in vmconfig['state']:
            new_disks = []
            for disk in vmconfig['state']['disks']:
                path = False
                if 'disks' in vmconfig['current']:
                    for cdisk in vmconfig['current']['disks']:
                        if cdisk['path'].endswith(disk['path']):
                            path = cdisk['path']
                            break
                if not path:
                    del disk['path']
                else:
                    disk['path'] = path
                new_disks.append(disk)
            vmconfig['state']['disks'] = new_disks

        # process properties
        for prop in vmconfig['state']:
            # skip special vmconfig_types
            if prop in vmconfig_type['instance'] or \
                prop in vmconfig_type['collection'] or \
                prop in vmconfig_type['create_only']:
                continue

            # skip unchanged properties
            if prop in vmconfig['current']:
                if isinstance(vmconfig['current'][prop], (list)) or isinstance(vmconfig['current'][prop], (dict)):
                    if vmconfig['current'][prop] == vmconfig['state'][prop]:
                        continue
                else:
                    if "{0}".format(vmconfig['current'][prop]) == "{0}".format(vmconfig['state'][prop]):
                        continue

            # add property to changeset
            vmconfig['changed'][prop] = vmconfig['state'][prop]

        # process collections
        for collection in vmconfig_type['collection']:
            # skip create only collections
            if collection in vmconfig_type['create_only']:
                continue

            # enforcement
            enforce = config['enforce_{0}'.format(collection)]
            log.debug('smartos.vm_present::enforce_%s = %s', collection, enforce)

            # dockerinit handling
            if collection == 'internal_metadata' and vmconfig['state'].get('docker', False):
                if 'internal_metadata' not in vmconfig['state']:
                    vmconfig['state']['internal_metadata'] = {}

                # preserve some docker specific metadata (added and needed by dockerinit)
                for var in vmconfig_docker_keep:
                    val = vmconfig['current'].get(collection, {}).get(var, None)
                    if val is not None:
                        vmconfig['state']['internal_metadata'][var] = val

            # process add and update for collection
            if collection in vmconfig['state'] and vmconfig['state'][collection] is not None:
                for prop in vmconfig['state'][collection]:
                    # skip unchanged properties
                    if prop in vmconfig['current'][collection] and \
                        vmconfig['current'][collection][prop] == vmconfig['state'][collection][prop]:
                        continue

                    # skip update if not enforcing
                    if not enforce and prop in vmconfig['current'][collection]:
                        continue

                    # create set_ dict
                    if 'set_{0}'.format(collection) not in vmconfig['changed']:
                        vmconfig['changed']['set_{0}'.format(collection)] = {}

                    # add property to changeset
                    vmconfig['changed']['set_{0}'.format(collection)][prop] = vmconfig['state'][collection][prop]

            # process remove for collection
            if enforce and collection in vmconfig['current'] and vmconfig['current'][collection] is not None:
                for prop in vmconfig['current'][collection]:
                    # skip if exists in state
                    if collection in vmconfig['state'] and vmconfig['state'][collection] is not None:
                        if prop in vmconfig['state'][collection]:
                            continue

                    # create remove_ array
                    if 'remove_{0}'.format(collection) not in vmconfig['changed']:
                        vmconfig['changed']['remove_{0}'.format(collection)] = []

                    # remove property
                    vmconfig['changed']['remove_{0}'.format(collection)].append(prop)

        # process instances
        for instance in vmconfig_type['instance']:
            # skip create only instances
            if instance in vmconfig_type['create_only']:
                continue

            # add or update instances
            if instance in vmconfig['state'] and vmconfig['state'][instance] is not None:
                for state_cfg in vmconfig['state'][instance]:
                    add_instance = True

                    # find instance with matching ids
                    for current_cfg in vmconfig['current'][instance]:
                        if vmconfig_type['instance'][instance] not in state_cfg:
                            continue

                        if state_cfg[vmconfig_type['instance'][instance]] == current_cfg[vmconfig_type['instance'][instance]]:
                            # ids have matched, disable add instance
                            add_instance = False

                            changed = _get_instance_changes(current_cfg, state_cfg)
                            update_cfg = {}

                            # handle changes
                            for prop in changed:
                                update_cfg[prop] = state_cfg[prop]

                            # handle new properties
                            for prop in state_cfg:
                                # skip empty props like ips, options,..
                                if isinstance(state_cfg[prop], (list)) and not state_cfg[prop]:
                                    continue

                                if prop not in current_cfg:
                                    update_cfg[prop] = state_cfg[prop]

                            # update instance
                            if update_cfg:
                                # create update_ array
                                if 'update_{0}'.format(instance) not in vmconfig['changed']:
                                    vmconfig['changed']['update_{0}'.format(instance)] = []

                                update_cfg[vmconfig_type['instance'][instance]] = state_cfg[vmconfig_type['instance'][instance]]
                                vmconfig['changed']['update_{0}'.format(instance)].append(update_cfg)

                    if add_instance:
                        # create add_ array
                        if 'add_{0}'.format(instance) not in vmconfig['changed']:
                            vmconfig['changed']['add_{0}'.format(instance)] = []

                        # add instance
                        vmconfig['changed']['add_{0}'.format(instance)].append(state_cfg)

            # remove instances
            if instance in vmconfig['current'] and vmconfig['current'][instance] is not None:
                for current_cfg in vmconfig['current'][instance]:
                    remove_instance = True

                    # find instance with matching ids
                    if instance in vmconfig['state'] and vmconfig['state'][instance] is not None:
                        for state_cfg in vmconfig['state'][instance]:
                            if vmconfig_type['instance'][instance] not in state_cfg:
                                continue

                            if state_cfg[vmconfig_type['instance'][instance]] == current_cfg[vmconfig_type['instance'][instance]]:
                                # keep instance if matched
                                remove_instance = False

                    if remove_instance:
                        # create remove_ array
                        if 'remove_{0}'.format(instance) not in vmconfig['changed']:
                            vmconfig['changed']['remove_{0}'.format(instance)] = []

                        # remove instance
                        vmconfig['changed']['remove_{0}'.format(instance)].append(
                            current_cfg[vmconfig_type['instance'][instance]]
                        )

        # update vm if we have pending changes
        kvm_needs_start = False
        if not __opts__['test'] and vmconfig['changed']:
            # stop kvm if disk updates and kvm_reboot
            if vmconfig['current']['brand'] == 'kvm' and config['kvm_reboot']:
                if 'add_disks' in vmconfig['changed'] or \
                    'update_disks' in vmconfig['changed'] or \
                    'remove_disks' in vmconfig['changed']:
                    if vmconfig['state']['hostname'] in __salt__['vmadm.list'](order='hostname', search='state=running'):
                        kvm_needs_start = True
                        __salt__['vmadm.stop'](vm=vmconfig['state']['hostname'], key='hostname')

            # do update
            rret = __salt__['vmadm.update'](vm=vmconfig['state']['hostname'], key='hostname', **vmconfig['changed'])
            if not isinstance(rret, (bool)) and 'Error' in rret:
                ret['result'] = False
                ret['comment'] = "{0}".format(rret['Error'])
            else:
                ret['result'] = True
                ret['changes'][vmconfig['state']['hostname']] = vmconfig['changed']

        if ret['result']:
            if __opts__['test']:
                ret['changes'][vmconfig['state']['hostname']] = vmconfig['changed']

            if vmconfig['state']['hostname'] in ret['changes'] and ret['changes'][vmconfig['state']['hostname']]:
                ret['comment'] = 'vm {0} updated'.format(vmconfig['state']['hostname'])
                if config['kvm_reboot'] and vmconfig['current']['brand'] == 'kvm' and not __opts__['test']:
                    if vmconfig['state']['hostname'] in __salt__['vmadm.list'](order='hostname', search='state=running'):
                        __salt__['vmadm.reboot'](vm=vmconfig['state']['hostname'], key='hostname')
                    if kvm_needs_start:
                        __salt__['vmadm.start'](vm=vmconfig['state']['hostname'], key='hostname')
            else:
                ret['changes'] = {}
                ret['comment'] = 'vm {0} is up to date'.format(vmconfig['state']['hostname'])

            # reprovision (if required and allowed)
            if 'image_uuid' in vmconfig['current'] and vmconfig['reprovision_uuid'] != vmconfig['current']['image_uuid']:
                if config['reprovision']:
                    rret = __salt__['vmadm.reprovision'](
                        vm=vmconfig['state']['hostname'],
                        key='hostname',
                        image=vmconfig['reprovision_uuid']
                    )
                    if not isinstance(rret, (bool)) and 'Error' in rret:
                        ret['result'] = False
                        ret['comment'] = 'vm {0} updated, reprovision failed'.format(
                            vmconfig['state']['hostname']
                        )
                    else:
                        ret['comment'] = 'vm {0} updated and reprovisioned'.format(vmconfig['state']['hostname'])
                        if vmconfig['state']['hostname'] not in ret['changes']:
                            ret['changes'][vmconfig['state']['hostname']] = {}
                        ret['changes'][vmconfig['state']['hostname']]['image_uuid'] = vmconfig['reprovision_uuid']
                else:
                    log.warning('smartos.vm_present::%s::reprovision - '
                                'image_uuid in state does not match current, '
                                'reprovision not allowed',
                                name)
        else:
            ret['comment'] = 'vm {0} failed to be updated'.format(vmconfig['state']['hostname'])
            if not isinstance(rret, (bool)) and 'Error' in rret:
                ret['comment'] = "{0}".format(rret['Error'])
    else:
        # check required image installed
        ret['result'] = True

        # disks need some special care
        if 'disks' in vmconfig:
            new_disks = []
            for disk in vmconfig['disks']:
                if 'path' in disk:
                    del disk['path']
                new_disks.append(disk)
            vmconfig['disks'] = new_disks

        # create vm
        if ret['result']:
            uuid = __salt__['vmadm.create'](**vmconfig) if not __opts__['test'] else True
            if not isinstance(uuid, (bool)) and 'Error' in uuid:
                ret['result'] = False
                ret['comment'] = "{0}".format(uuid['Error'])
            else:
                ret['result'] = True
                ret['changes'][vmconfig['hostname']] = vmconfig
                ret['comment'] = 'vm {0} created'.format(vmconfig['hostname'])

    return ret

Response: Ensure vm is present on the computenode


Instruction:
Implement a Python 3 function for
ensuring vm is absent on the computenode

Response:
def vm_absent(name, archive=False):
    '''
    Ensure vm is absent on the computenode

    name : string
        hostname of vm
    archive : boolean
        toggle archiving of vm on removal

    .. note::

        State ID is used as hostname. Hostnames must be unique.

    '''
    name = name.lower()
    ret = {'name': name,
           'changes': {},
           'result': None,
           'comment': ''}

    if name not in __salt__['vmadm.list'](order='hostname'):
        # we're good
        ret['result'] = True
        ret['comment'] = 'vm {0} is absent'.format(name)
    else:
        # delete vm
        if not __opts__['test']:
            # set archive to true if needed
            if archive:
                __salt__['vmadm.update'](vm=name, key='hostname', archive_on_delete=True)

            ret['result'] = __salt__['vmadm.delete'](name, key='hostname')
        else:
            ret['result'] = True

        if not isinstance(ret['result'], bool) and ret['result'].get('Error'):
            ret['result'] = False
            ret['comment'] = 'failed to delete vm {0}'.format(name)
        else:
            ret['comment'] = 'vm {0} deleted'.format(name)
            ret['changes'][name] = None

    return ret


Instruction:
Create a Python 3 function to
ensure vm is in the running state on the computenode

Response:
def vm_running(name):
    '''
    Ensure vm is in the running state on the computenode

    name : string
        hostname of vm

    .. note::

        State ID is used as hostname. Hostnames must be unique.

    '''
    name = name.lower()
    ret = {'name': name,
           'changes': {},
           'result': None,
           'comment': ''}

    if name in __salt__['vmadm.list'](order='hostname', search='state=running'):
        # we're good
        ret['result'] = True
        ret['comment'] = 'vm {0} already running'.format(name)
    else:
        # start the vm
        ret['result'] = True if __opts__['test'] else __salt__['vmadm.start'](name, key='hostname')
        if not isinstance(ret['result'], bool) and ret['result'].get('Error'):
            ret['result'] = False
            ret['comment'] = 'failed to start {0}'.format(name)
        else:
            ret['changes'][name] = 'running'
            ret['comment'] = 'vm {0} started'.format(name)

    return ret


Instruction:
Can you generate the documentation for the following Python 3 function
def _repack_options(options):
    '''
    Repack the options data
    '''
    return dict(
        [
            (six.text_type(x), _normalize(y))
            for x, y in six.iteritems(salt.utils.data.repack_dictlist(options))
        ]
    )

Response: Repack the options data


Instruction:
Make a summary of the following Python 3 code
def _get_option_list(options):
    '''
    Returns the key/value pairs in the passed dict in a commaspace-delimited
    list in the format "key=value".
    '''
    return ', '.join(['{0}={1}'.format(x, y) for x, y in six.iteritems(options)])

Response:
Returns the key = value pairs in the passed dict in a commaspace - delimited
    list in the format key = value.


Instruction:
Can you create a Python 3 function that
verifies that the port is installed and that it has the specified options.

Response:
def installed(name, options=None):
    '''
    Verify that the desired port is installed, and that it was compiled with
    the desired options.

    options
        Make sure that the desired non-default options are set

        .. warning::

            Any build options not passed here assume the default values for the
            port, and are not just differences from the existing cached options
            from a previous ``make config``.

    Example usage:

    .. code-block:: yaml

        security/nmap:
          ports.installed:
            - options:
              - IPV6: off
    '''
    ret = {'name': name,
           'changes': {},
           'result': True,
           'comment': '{0} is already installed'.format(name)}
    try:
        current_options = __salt__['ports.showconfig'](name, default=False,
                                                       dict_return=True)
        default_options = __salt__['ports.showconfig'](name, default=True,
                                                       dict_return=True)
        # unpack the options from the top-level return dict
        if current_options:
            current_options = current_options[next(iter(current_options))]
        if default_options:
            default_options = default_options[next(iter(default_options))]
    except (SaltInvocationError, CommandExecutionError) as exc:
        ret['result'] = False
        ret['comment'] = ('Unable to get configuration for {0}. Port name may '
                          'be invalid, or ports tree may need to be updated. '
                          'Error message: {1}'.format(name, exc))
        return ret

    options = _repack_options(options) if options is not None else {}
    desired_options = copy.deepcopy(default_options)
    desired_options.update(options)
    ports_pre = [
        x['origin'] for x in
        six.itervalues(__salt__['pkg.list_pkgs'](with_origin=True))
    ]

    if current_options == desired_options and name in ports_pre:
        # Port is installed as desired
        if options:
            ret['comment'] += ' ' + _build_option_string(options)
        return ret

    if not default_options:
        if options:
            ret['result'] = False
            ret['comment'] = ('{0} does not have any build options, yet '
                              'options were specified'.format(name))
            return ret
        else:
            if __opts__['test']:
                ret['result'] = None
                ret['comment'] = '{0} will be installed'.format(name)
                return ret
    else:
        bad_opts = [x for x in options if x not in default_options]
        if bad_opts:
            ret['result'] = False
            ret['comment'] = ('The following options are not available for '
                              '{0}: {1}'.format(name, ', '.join(bad_opts)))
            return ret

        if __opts__['test']:
            ret['result'] = None
            ret['comment'] = '{0} will be installed '.format(name)
            ret['comment'] += _build_option_string(options)
            return ret

        if options:
            if not __salt__['ports.config'](name, reset=True, **options):
                ret['result'] = False
                ret['comment'] = 'Unable to set options for {0}'.format(name)
                return ret
        else:
            __salt__['ports.rmconfig'](name)
            if _options_file_exists(name):
                ret['result'] = False
                ret['comment'] = 'Unable to clear options for {0}'.format(name)
                return ret

    ret['changes'] = __salt__['ports.install'](name)
    ports_post = [
        x['origin'] for x in
        six.itervalues(__salt__['pkg.list_pkgs'](with_origin=True))
    ]
    err = sys.modules[
        __salt__['test.ping'].__module__
    ].__context__.pop('ports.install_error', None)
    if err or name not in ports_post:
        ret['result'] = False
    if ret['result']:
        ret['comment'] = 'Successfully installed {0}'.format(name)
        if default_options:
            ret['comment'] += ' ' + _build_option_string(options)
    else:
        ret['comment'] = 'Failed to install {0}'.format(name)
        if err:
            ret['comment'] += '. Error message:\n{0}'.format(err)
    return ret


Instruction:
Write a Python 3 function for
validating the configuration passed to the API.

Response:
def _validate(dns_proto, dns_servers, ip_proto, ip_addrs, gateway):
    '''
    Ensure that the configuration passed is formatted correctly and contains
    valid IP addresses, etc.
    '''
    errors = []
    # Validate DNS configuration
    if dns_proto == 'dhcp':
        if dns_servers is not None:
            errors.append(
                'The dns_servers param cannot be set if unless dns_proto is '
                'set to \'static\''
            )
    else:
        if str(dns_servers).lower() in ['none', '[]']:
            pass
        elif not isinstance(dns_servers, list):
            errors.append(
                'The dns_servers param must be formatted as a list'
            )
        else:
            bad_ips = [x for x in dns_servers
                       if not salt.utils.validate.net.ipv4_addr(x)]
            if bad_ips:
                errors.append('Invalid DNS server IPs: {0}'
                              .format(', '.join(bad_ips)))

    # Validate IP configuration
    if ip_proto == 'dhcp':
        if ip_addrs is not None:
            errors.append(
                'The ip_addrs param cannot be set if unless ip_proto is set '
                'to \'static\''
            )
        if gateway is not None:
            errors.append(
                'A gateway IP cannot be set if unless ip_proto is set to '
                '\'static\''
            )
    else:
        if not ip_addrs:
            errors.append(
                'The ip_addrs param is required to set static IPs'
            )
        elif not isinstance(ip_addrs, list):
            errors.append(
                'The ip_addrs param must be formatted as a list'
            )
        else:
            bad_ips = [x for x in ip_addrs
                       if not salt.utils.validate.net.ipv4_addr(x)]
            if bad_ips:
                errors.append('The following static IPs are invalid: '
                              '{0}'.format(', '.join(bad_ips)))

            # Validate default gateway
            if gateway is not None:
                if not salt.utils.validate.net.ipv4_addr(gateway):
                    errors.append('Gateway IP {0} is invalid'.format(gateway))

    return errors


Instruction:
Make a summary of the following Python 3 code
def _changes(cur, dns_proto, dns_servers, ip_proto, ip_addrs, gateway):
    '''
    Compares the current interface against the desired configuration and
    returns a dictionary describing the changes that need to be made.
    '''
    changes = {}
    cur_dns_proto = (
        'static' if 'Statically Configured DNS Servers' in cur
        else 'dhcp'
    )
    if cur_dns_proto == 'static':
        if isinstance(cur['Statically Configured DNS Servers'], list):
            cur_dns_servers = cur['Statically Configured DNS Servers']
        else:
            cur_dns_servers = [cur['Statically Configured DNS Servers']]
        if set(dns_servers or ['None']) != set(cur_dns_servers):
            changes['dns_servers'] = dns_servers
    elif 'DNS servers configured through DHCP' in cur:
        cur_dns_servers = cur['DNS servers configured through DHCP']
        if dns_proto == 'static':
            # If we're currently set to 'dhcp' but moving to 'static', specify the changes.
            if set(dns_servers or ['None']) != set(cur_dns_servers):
                changes['dns_servers'] = dns_servers

    cur_ip_proto = 'static' if cur['DHCP enabled'] == 'No' else 'dhcp'
    cur_ip_addrs = _addrdict_to_ip_addrs(cur.get('ip_addrs', []))
    cur_gateway = cur.get('Default Gateway')

    if dns_proto != cur_dns_proto:
        changes['dns_proto'] = dns_proto
    if ip_proto != cur_ip_proto:
        changes['ip_proto'] = ip_proto
    if set(ip_addrs or []) != set(cur_ip_addrs):
        if ip_proto == 'static':
            changes['ip_addrs'] = ip_addrs
    if gateway != cur_gateway:
        if ip_proto == 'static':
            changes['gateway'] = gateway
    return changes

Response: Returns a dictionary describing the changes made to the current configuration and the current interface.


Instruction:
Explain what the following Python 3 code does
def managed(name,
            dns_proto=None,
            dns_servers=None,
            ip_proto=None,
            ip_addrs=None,
            gateway=None,
            enabled=True,
            **kwargs):
    '''
    Ensure that the named interface is configured properly.

    Args:

        name (str):
            The name of the interface to manage

        dns_proto (str): None
            Set to ``static`` and use the ``dns_servers`` parameter to provide a
            list of DNS nameservers. set to ``dhcp`` to use DHCP to get the DNS
            servers.

        dns_servers (list): None
            A list of static DNS servers. To clear the list of DNS servers pass
            an empty list (``[]``). ``None`` will make no changes.

        ip_proto (str): None
            Set to ``static`` and use the ``ip_addrs`` and (optionally)
            ``gateway`` parameters to provide a list of static IP addresses and
            the default gateway. Set to ``dhcp`` to use DHCP.

        ip_addrs (list): None
            A list of static IP addresses with netmask flag, ie: 192.168.0.11/24

        gateway (str): None
            The gateway to set for the interface

        enabled (bool): True
            Set to ``False`` to ensure that this interface is disabled.

    Returns:
        dict: A dictionary of old and new settings

    Example:

    .. code-block:: yaml

        Ethernet1:
          network.managed:
            - dns_proto: static
            - dns_servers:
              - 8.8.8.8
              - 8.8.8.4
            - ip_proto: static
            - ip_addrs:
              - 192.168.0.100/24

    Clear DNS entries example:

    .. code-block:: yaml

        Ethernet1:
          network.managed:
            - dns_proto: static
            - dns_servers: []
            - ip_proto: dhcp
    '''
    ret = {
        'name': name,
        'changes': {},
        'result': True,
        'comment': 'Interface \'{0}\' is up to date'.format(name)
    }

    dns_proto = six.text_type(dns_proto).lower()
    ip_proto = six.text_type(ip_proto).lower()

    errors = []
    if dns_proto not in __VALID_PROTO:
        ret['result'] = False
        errors.append('dns_proto must be one of the following: {0}'
                      .format(', '.join(__VALID_PROTO)))

    if ip_proto not in __VALID_PROTO:
        errors.append('ip_proto must be one of the following: {0}'
                      .format(', '.join(__VALID_PROTO)))

    if errors:
        ret['result'] = False
        ret['comment'] = '\n'.join(errors)
        return ret

    try:
        currently_enabled = __salt__['ip.is_enabled'](name)
    except CommandExecutionError:
        currently_enabled = False

    if not enabled:
        if currently_enabled:
            if __opts__['test']:
                ret['result'] = None
                ret['comment'] = ('Interface \'{0}\' will be disabled'
                                  .format(name))
            else:
                ret['result'] = __salt__['ip.disable'](name)
                if not ret['result']:
                    ret['comment'] = ('Failed to disable interface \'{0}\''
                                      .format(name))
        else:
            ret['comment'] += ' (already disabled)'
        return ret
    else:
        if not currently_enabled:
            if __opts__['test']:
                ret['result'] = None
                ret['comment'] = ('Interface \'{0}\' will be enabled'
                                  .format(name))
            else:
                if not __salt__['ip.enable'](name):
                    ret['result'] = False
                    ret['comment'] = ('Failed to enable interface \'{0}\' to '
                                      'make changes'.format(name))
                    return ret

        errors = _validate(dns_proto, dns_servers, ip_proto, ip_addrs, gateway)
        if errors:
            ret['result'] = False
            ret['comment'] = ('The following SLS configuration errors were '
                              'detected:\n- {0}'.format('\n- '.join(errors)))
            return ret

        old = __salt__['ip.get_interface'](name)
        if not old:
            ret['result'] = False
            ret['comment'] = ('Unable to get current configuration for '
                              'interface \'{0}\''.format(name))
            return ret

        changes = _changes(old,
                           dns_proto,
                           dns_servers,
                           ip_proto,
                           ip_addrs,
                           gateway)

        # If dns_servers is the default `None` make no changes
        # To clear the list, pass an empty dict
        if str(dns_servers).lower() == 'none':
            changes.pop('dns_servers', None)

        if not changes:
            return ret

        if __opts__['test']:
            comments = []
            if 'dns_proto' in changes:
                comments.append('DNS protocol will be changed to: {0}'
                                .format(changes['dns_proto']))
            if dns_proto == 'static' and 'dns_servers' in changes:
                if not changes['dns_servers']:
                    comments.append('The list of DNS servers will be cleared')
                else:
                    comments.append(
                        'DNS servers will be set to the following: {0}'
                        .format(', '.join(changes['dns_servers']))
                    )
            if 'ip_proto' in changes:
                comments.append('IP protocol will be changed to: {0}'
                                .format(changes['ip_proto']))
            if ip_proto == 'static':
                if 'ip_addrs' in changes:
                    comments.append(
                        'IP addresses will be set to the following: {0}'
                        .format(', '.join(changes['ip_addrs']))
                    )
                if 'gateway' in changes:
                    if changes['gateway'] is None:
                        comments.append('Default gateway will be removed')
                    else:
                        comments.append(
                            'Default gateway will be set to {0}'
                            .format(changes['gateway'])
                        )

            ret['result'] = None
            ret['comment'] = ('The following changes will be made to '
                              'interface \'{0}\':\n- {1}'
                              .format(name, '\n- '.join(comments)))
            return ret

        if changes.get('dns_proto') == 'dhcp':
            __salt__['ip.set_dhcp_dns'](name)

        elif 'dns_servers' in changes:
            if not changes['dns_servers']:
                # To clear the list of DNS servers you have to pass []. Later
                # changes gets passed like *args and a single empty list is
                # converted to an empty tuple. So, you have to add [] here
                changes['dns_servers'] = [[]]

            __salt__['ip.set_static_dns'](name, *changes['dns_servers'])

        if changes.get('ip_proto') == 'dhcp':
            __salt__['ip.set_dhcp_ip'](name)
        elif changes.get('ip_addrs') or changes.get('gateway') or changes.get('ip_proto') == 'static':
            if changes.get('gateway') and not changes.get('ip_addrs'):
                changes['ip_addrs'] = ip_addrs
            if changes.get('ip_proto') == 'static' and not changes.get('ip_addrs'):
                changes['ip_addrs'] = ip_addrs
            for idx in range(len(changes['ip_addrs'])):
                if idx == 0:
                    __salt__['ip.set_static_ip'](
                        name,
                        changes['ip_addrs'][idx],
                        gateway=gateway,
                        append=False
                    )
                else:
                    __salt__['ip.set_static_ip'](
                        name,
                        changes['ip_addrs'][idx],
                        gateway=None,
                        append=True
                    )

        new = __salt__['ip.get_interface'](name)
        ret['changes'] = salt.utils.data.compare_dicts(old, new)
        if _changes(new, dns_proto, dns_servers, ip_proto, ip_addrs, gateway):
            ret['result'] = False
            ret['comment'] = ('Failed to set desired configuration settings '
                              'for interface \'{0}\''.format(name))
        else:
            ret['comment'] = ('Successfully updated configuration for '
                              'interface \'{0}\''.format(name))
        return ret

Response: This function manage the named interface with the specified DNS servers and IP addresses.


Instruction:
Can you implement a function in Python 3 that
sets up libcrypto argtypes and initialize the library

Response:
def _init_libcrypto():
    '''
    Set up libcrypto argtypes and initialize the library
    '''
    libcrypto = _load_libcrypto()

    try:
        libcrypto.OPENSSL_init_crypto()
    except AttributeError:
        # Support for OpenSSL < 1.1 (OPENSSL_API_COMPAT < 0x10100000L)
        libcrypto.OPENSSL_no_config()
        libcrypto.OPENSSL_add_all_algorithms_noconf()

    libcrypto.RSA_new.argtypes = ()
    libcrypto.RSA_new.restype = c_void_p
    libcrypto.RSA_free.argtypes = (c_void_p, )
    libcrypto.RSA_size.argtype = (c_void_p)
    libcrypto.BIO_new_mem_buf.argtypes = (c_char_p, c_int)
    libcrypto.BIO_new_mem_buf.restype = c_void_p
    libcrypto.BIO_free.argtypes = (c_void_p, )
    libcrypto.PEM_read_bio_RSAPrivateKey.argtypes = (c_void_p, c_void_p, c_void_p, c_void_p)
    libcrypto.PEM_read_bio_RSAPrivateKey.restype = c_void_p
    libcrypto.PEM_read_bio_RSA_PUBKEY.argtypes = (c_void_p, c_void_p, c_void_p, c_void_p)
    libcrypto.PEM_read_bio_RSA_PUBKEY.restype = c_void_p
    libcrypto.RSA_private_encrypt.argtypes = (c_int, c_char_p, c_char_p, c_void_p, c_int)
    libcrypto.RSA_public_decrypt.argtypes = (c_int, c_char_p, c_char_p, c_void_p, c_int)

    return libcrypto


Instruction:
Here you have a function in Python 3, explain what it does
def sign(self, msg):
        '''
        Sign a message (digest) using the private key

        :param str msg: The message (digest) to sign
        :rtype: str
        :return: The signature, or an empty string if the encryption failed
        '''
        # Allocate a buffer large enough for the signature. Freed by ctypes.
        buf = create_string_buffer(libcrypto.RSA_size(self._rsa))
        msg = salt.utils.stringutils.to_bytes(msg)
        size = libcrypto.RSA_private_encrypt(len(msg), msg, buf, self._rsa, RSA_X931_PADDING)
        if size < 0:
            raise ValueError('Unable to encrypt message')
        return buf[0:size]

Response: Sign a message using the private key.


Instruction:
How would you implement a function in Python 3 that
verifies the message from the signature using the public key.

Response:
def verify(self, signed):
        '''
        Recover the message (digest) from the signature using the public key

        :param str signed: The signature created with the private key
        :rtype: str
        :return: The message (digest) recovered from the signature, or an empty
            string if the decryption failed
        '''
        # Allocate a buffer large enough for the signature. Freed by ctypes.
        buf = create_string_buffer(libcrypto.RSA_size(self._rsa))
        signed = salt.utils.stringutils.to_bytes(signed)
        size = libcrypto.RSA_public_decrypt(len(signed), signed, buf, self._rsa, RSA_X931_PADDING)
        if size < 0:
            raise ValueError('Unable to decrypt message')
        return buf[0:size]


Instruction:
Here you have a function in Python 3, explain what it does
def install(pkg=None,
            pkgs=None,
            user=None,
            install_global=False,
            env=None):
    '''
    Install a cabal package.

    pkg
        A package name in format accepted by cabal-install. See:
        https://wiki.haskell.org/Cabal-Install

    pkgs
        A list of packages names in same format as ``pkg``

    user
        The user to run cabal install with

    install_global
        Install package globally instead of locally

    env
        Environment variables to set when invoking cabal. Uses the
        same ``env`` format as the :py:func:`cmd.run
        <salt.modules.cmdmod.run>` execution function

    CLI Example:

    .. code-block:: bash

        salt '*' cabal.install shellcheck
        salt '*' cabal.install shellcheck-0.3.5
    '''

    cmd = ['cabal install']

    if install_global:
        cmd.append('--global')

    if pkg:
        cmd.append('"{0}"'.format(pkg))
    elif pkgs:
        cmd.append('"{0}"'.format('" "'.join(pkgs)))

    result = __salt__['cmd.run_all'](' '.join(cmd), runas=user, env=env)

    if result['retcode'] != 0:
        raise CommandExecutionError(result['stderr'])

    return result

Response: Install a cabal package.


Instruction:
Make a summary of the following Python 3 code
def list_(
        pkg=None,
        user=None,
        installed=False,
        env=None):
    '''
    List packages matching a search string.

    pkg
        Search string for matching package names
    user
        The user to run cabal list with
    installed
        If True, only return installed packages.
    env
        Environment variables to set when invoking cabal. Uses the
        same ``env`` format as the :py:func:`cmd.run
        <salt.modules.cmdmod.run>` execution function

    CLI example:

    .. code-block:: bash

        salt '*' cabal.list
        salt '*' cabal.list ShellCheck
    '''
    cmd = ['cabal list --simple-output']

    if installed:
        cmd.append('--installed')

    if pkg:
        cmd.append('"{0}"'.format(pkg))

    result = __salt__['cmd.run_all'](' '.join(cmd), runas=user, env=env)

    packages = {}
    for line in result['stdout'].splitlines():
        data = line.split()
        package_name = data[0]
        package_version = data[1]
        packages[package_name] = package_version

    return packages

Response: List the packages matching a search string.


Instruction:
Can you write a function in Python 3 where it
takes a string or file object and runs it through the YAML_EX parser and returns a Python data structure

Response:
def render(sls_data, saltenv='base', sls='', **kws):
    '''
    Accepts YAML_EX as a string or as a file object and runs it through the YAML_EX
    parser.

    :rtype: A Python data structure
    '''
    with warnings.catch_warnings(record=True) as warn_list:
        data = deserialize(sls_data) or {}

        for item in warn_list:
            log.warning(
                '%s found in %s saltenv=%s',
                item.message, salt.utils.url.create(sls), saltenv
            )

        log.debug('Results of SLS rendering: \n%s', data)

    return data


Instruction:
Can you generate the documentation for the following Python 3 function
def returner(ret):
    '''
    Signal a Django server that a return is available
    '''
    signaled = dispatch.Signal(providing_args=['ret']).send(sender='returner', ret=ret)

    for signal in signaled:
        log.debug(
            'Django returner function \'returner\' signaled %s '
            'which responded with %s', signal[0], signal[1]
        )

Response: Signal a Django server that a return is available on a resource.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def save_load(jid, load, minions=None):
    '''
    Save the load to the specified jid
    '''
    signaled = dispatch.Signal(
        providing_args=['jid', 'load']).send(
            sender='save_load', jid=jid, load=load)

    for signal in signaled:
        log.debug(
            'Django returner function \'save_load\' signaled %s '
            'which responded with %s', signal[0], signal[1]
        )

Response: Save the load to the specified jid


Instruction:
How would you explain what the following Python 3 function does
def prep_jid(nocache=False, passed_jid=None):
    '''
    Do any work necessary to prepare a JID, including sending a custom ID
    '''
    return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid(__opts__)

Response: Prepare a JID for sending to the master


Instruction:
Can you write a function in Python 3 where it
loads and start all available netapi modules

Response:
def run(self):
        '''
        Load and start all available api modules
        '''
        if not len(self.netapi):
            log.error("Did not find any netapi configurations, nothing to start")

        kwargs = {}
        if salt.utils.platform.is_windows():
            kwargs['log_queue'] = salt.log.setup.get_multiprocessing_logging_queue()
            kwargs['log_queue_level'] = salt.log.setup.get_multiprocessing_logging_level()

        for fun in self.netapi:
            if fun.endswith('.start'):
                log.info('Starting %s netapi module', fun)
                self.process_manager.add_process(
                    RunNetapi,
                    args=(self.opts, fun),
                    kwargs=kwargs,
                    name='RunNetapi'
                )

        # Install the SIGINT/SIGTERM handlers if not done so far
        if signal.getsignal(signal.SIGINT) is signal.SIG_DFL:
            # No custom signal handling was added, install our own
            signal.signal(signal.SIGINT, self._handle_signals)

        if signal.getsignal(signal.SIGTERM) is signal.SIG_DFL:
            # No custom signal handling was added, install our own
            signal.signal(signal.SIGTERM, self._handle_signals)

        self.process_manager.run()


Instruction:
Make a summary of the following Python 3 code
def top(**kwargs):
    '''
    Query |varstack| for the top data (states of the minions).
    '''

    conf = __opts__['master_tops']['varstack']
    __grains__ = kwargs['grains']

    vs_ = varstack.Varstack(config_filename=conf)
    ret = vs_.evaluate(__grains__)
    return {'base': ret['states']}

Response: Query |varstack| for the top data of the minions.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_value(file, element):
    '''
    Returns the value of the matched xpath element

    CLI Example:

    .. code-block:: bash

        salt '*' xml.get_value /tmp/test.xml ".//element"
    '''
    try:
        root = ET.parse(file)
        element = root.find(element)
        return element.text
    except AttributeError:
        log.error("Unable to find element matching %s", element)
        return False

Response: Get the value of the matched xpath element


Instruction:
Implement a function in Python 3 to
set the value of the matched xpath element

Response:
def set_value(file, element, value):
    '''
    Sets the value of the matched xpath element

    CLI Example:

    .. code-block:: bash

        salt '*' xml.set_value /tmp/test.xml ".//element" "new value"
    '''
    try:
        root = ET.parse(file)
        relement = root.find(element)
    except AttributeError:
        log.error("Unable to find element matching %s", element)
        return False
    relement.text = str(value)
    root.write(file)
    return True


Instruction:
How would you code a function in Python 3 to
get the attributes of the matched xpath element.

Response:
def get_attribute(file, element):
    '''
    Return the attributes of the matched xpath element.

    CLI Example:

    .. code-block:: bash

        salt '*' xml.get_attribute /tmp/test.xml ".//element[@id='3']"
    '''
    try:
        root = ET.parse(file)
        element = root.find(element)
        return element.attrib
    except AttributeError:
        log.error("Unable to find element matching %s", element)
        return False


Instruction:
Write a Python 3 function for
setting the requested attribute key and value for matched xpath element.

Response:
def set_attribute(file, element, key, value):
    '''
    Set the requested attribute key and value for matched xpath element.

    CLI Example:

    .. code-block:: bash

        salt '*' xml.set_attribute /tmp/test.xml ".//element[@id='3']" editedby "gal"
    '''
    try:
        root = ET.parse(file)
        element = root.find(element)
    except AttributeError:
        log.error("Unable to find element matching %s", element)
        return False
    element.set(key, str(value))
    root.write(file)
    return True


Instruction:
How would you explain what the following Python 3 function does
def get_account(self):
        '''
        List Swift containers
        '''
        try:
            listing = self.conn.get_account()
            return listing
        except Exception as exc:
            log.error('There was an error::')
            if hasattr(exc, 'code') and hasattr(exc, 'msg'):
                log.error('    Code: %s: %s', exc.code, exc.msg)
            log.error('    Content: \n%s', getattr(exc, 'read', lambda: six.text_type(exc))())
            return False

Response: Get the current Swift containers from the server.


Instruction:
How would you code a function in Python 3 to
retrieve a file from Swift

Response:
def get_object(self, cont, obj, local_file=None, return_bin=False):
        '''
        Retrieve a file from Swift
        '''
        try:
            if local_file is None and return_bin is False:
                return False

            headers, body = self.conn.get_object(cont, obj, resp_chunk_size=65536)

            if return_bin is True:
                fp = sys.stdout
            else:
                dirpath = dirname(local_file)
                if dirpath and not isdir(dirpath):
                    mkdirs(dirpath)
                fp = salt.utils.files.fopen(local_file, 'wb')  # pylint: disable=resource-leakage

            read_length = 0
            for chunk in body:
                read_length += len(chunk)
                fp.write(chunk)
            fp.close()
            return True

        # ClientException
        # file/dir exceptions
        except Exception as exc:
            log.error('There was an error::')
            if hasattr(exc, 'code') and hasattr(exc, 'msg'):
                log.error('    Code: %s: %s', exc.code, exc.msg)
            log.error('    Content: \n%s', getattr(exc, 'read', lambda: six.text_type(exc))())
            return False


Instruction:
Make a summary of the following Python 3 code
def put_object(self, cont, obj, local_file):
        '''
        Upload a file to Swift
        '''
        try:
            with salt.utils.files.fopen(local_file, 'rb') as fp_:
                self.conn.put_object(cont, obj, fp_)
            return True
        except Exception as exc:
            log.error('There was an error::')
            if hasattr(exc, 'code') and hasattr(exc, 'msg'):
                log.error('    Code: %s: %s', exc.code, exc.msg)
            log.error('    Content: \n%s', getattr(exc, 'read', lambda: six.text_type(exc))())
            return False

Response: Uploads a file to Swift


Instruction:
Create a Python 3 function to
delete a file from Swift

Response:
def delete_object(self, cont, obj):
        '''
        Delete a file from Swift
        '''
        try:
            self.conn.delete_object(cont, obj)
            return True
        except Exception as exc:
            log.error('There was an error::')
            if hasattr(exc, 'code') and hasattr(exc, 'msg'):
                log.error('    Code: %s: %s', exc.code, exc.msg)
            log.error('    Content: \n%s', getattr(exc, 'read', lambda: six.text_type(exc))())
            return False


Instruction:
How would you explain what the following Python 3 function does
def _xfs_info_get_kv(serialized):
    '''
    Parse one line of the XFS info output.
    '''
    # No need to know sub-elements here
    if serialized.startswith("="):
        serialized = serialized[1:].strip()

    serialized = serialized.replace(" = ", "=*** ").replace(" =", "=")

    # Keywords has no spaces, values do
    opt = []
    for tkn in serialized.split(" "):
        if not opt or "=" in tkn:
            opt.append(tkn)
        else:
            opt[len(opt) - 1] = opt[len(opt) - 1] + " " + tkn

    # Preserve ordering
    return [tuple(items.split("=")) for items in opt]

Response: Parse one line of the XFS info output.


Instruction:
Implement a function in Python 3 to
parse output from xfs_info or xfs_growfs - n.

Response:
def _parse_xfs_info(data):
    '''
    Parse output from "xfs_info" or "xfs_growfs -n".
    '''
    ret = {}
    spr = re.compile(r'\s+')
    entry = None
    for line in [spr.sub(" ", l).strip().replace(", ", " ") for l in data.split("\n")]:
        if not line:
            continue
        nfo = _xfs_info_get_kv(line)
        if not line.startswith("="):
            entry = nfo.pop(0)
            ret[entry[0]] = {'section': entry[(entry[1] != '***' and 1 or 0)]}
        ret[entry[0]].update(dict(nfo))

    return ret


Instruction:
Here you have a function in Python 3, explain what it does
def info(device):
    '''
    Get filesystem geometry information.

    CLI Example:

    .. code-block:: bash

        salt '*' xfs.info /dev/sda1
    '''
    out = __salt__['cmd.run_all']("xfs_info {0}".format(device))
    if out.get('stderr'):
        raise CommandExecutionError(out['stderr'].replace("xfs_info:", "").strip())
    return _parse_xfs_info(out['stdout'])

Response: Get filesystem geometry information.


Instruction:
How would you explain what the following Python 3 function does
def _xfsdump_output(data):
    '''
    Parse CLI output of the xfsdump utility.
    '''
    out = {}
    summary = []
    summary_block = False

    for line in [l.strip() for l in data.split("\n") if l.strip()]:
        line = re.sub("^xfsdump: ", "", line)
        if line.startswith("session id:"):
            out['Session ID'] = line.split(" ")[-1]
        elif line.startswith("session label:"):
            out['Session label'] = re.sub("^session label: ", "", line)
        elif line.startswith("media file size"):
            out['Media size'] = re.sub(r"^media file size\s+", "", line)
        elif line.startswith("dump complete:"):
            out['Dump complete'] = re.sub(r"^dump complete:\s+", "", line)
        elif line.startswith("Dump Status:"):
            out['Status'] = re.sub(r"^Dump Status:\s+", "", line)
        elif line.startswith("Dump Summary:"):
            summary_block = True
            continue

        if line.startswith(" ") and summary_block:
            summary.append(line.strip())
        elif not line.startswith(" ") and summary_block:
            summary_block = False

    if summary:
        out['Summary'] = ' '.join(summary)

    return out

Response: Parse the CLI output of the xfsdump utility.


Instruction:
Write a Python 3 function that can
dump filesystem device to the media file.

Response:
def dump(device, destination, level=0, label=None, noerase=None):
    '''
    Dump filesystem device to the media (file, tape etc).

    Required parameters:

    * **device**: XFS device, content of which to be dumped.
    * **destination**: Specifies a dump destination.

    Valid options are:

    * **label**: Label of the dump. Otherwise automatically generated label is used.
    * **level**: Specifies a dump level of 0 to 9.
    * **noerase**: Pre-erase media.

    Other options are not used in order to let ``xfsdump`` use its default
    values, as they are most optimal. See the ``xfsdump(8)`` manpage for
    a more complete description of these options.

    CLI Example:

    .. code-block:: bash

        salt '*' xfs.dump /dev/sda1 /detination/on/the/client
        salt '*' xfs.dump /dev/sda1 /detination/on/the/client label='Company accountancy'
        salt '*' xfs.dump /dev/sda1 /detination/on/the/client noerase=True
    '''
    if not salt.utils.path.which("xfsdump"):
        raise CommandExecutionError("Utility \"xfsdump\" has to be installed or missing.")

    label = label and label or time.strftime("XFS dump for \"{0}\" of %Y.%m.%d, %H:%M".format(device),
                                             time.localtime()).replace("'", '"')
    cmd = ["xfsdump"]
    cmd.append("-F")                          # Force
    if not noerase:
        cmd.append("-E")                      # pre-erase
    cmd.append("-L '{0}'".format(label))      # Label
    cmd.append("-l {0}".format(level))        # Dump level
    cmd.append("-f {0}".format(destination))  # Media destination
    cmd.append(device)                        # Device

    cmd = ' '.join(cmd)
    out = __salt__['cmd.run_all'](cmd)
    _verify_run(out, cmd=cmd)

    return _xfsdump_output(out['stdout'])


Instruction:
Can you tell what is the following Python 3 function doing
def _xr_to_keyset(line):
    '''
    Parse xfsrestore output keyset elements.
    '''
    tkns = [elm for elm in line.strip().split(":", 1) if elm]
    if len(tkns) == 1:
        return "'{0}': ".format(tkns[0])
    else:
        key, val = tkns
        return "'{0}': '{1}',".format(key.strip(), val.strip())

Response: Parse xfsrestore output keyset elements.


Instruction:
Implement a Python 3 function for
transforming xfsrestore inventory data output to a Python dict source and evaluate it.

Response:
def _xfs_inventory_output(out):
    '''
    Transform xfsrestore inventory data output to a Python dict source and evaluate it.
    '''
    data = []
    out = [line for line in out.split("\n") if line.strip()]

    # No inventory yet
    if len(out) == 1 and 'restore status' in out[0].lower():
        return {'restore_status': out[0]}

    ident = 0
    data.append("{")
    for line in out[:-1]:
        if len([elm for elm in line.strip().split(":") if elm]) == 1:
            n_ident = len(re.sub("[^\t]", "", line))
            if ident > n_ident:
                for step in range(ident):
                    data.append("},")
            ident = n_ident
            data.append(_xr_to_keyset(line))
            data.append("{")
        else:
            data.append(_xr_to_keyset(line))
    for step in range(ident + 1):
        data.append("},")
    data.append("},")

    # We are evaling into a python dict, a json load
    # would be safer
    data = eval('\n'.join(data))[0]  # pylint: disable=W0123
    data['restore_status'] = out[-1]

    return data


Instruction:
Here you have a function in Python 3, explain what it does
def prune_dump(sessionid):
    '''
    Prunes the dump session identified by the given session id.

    CLI Example:

    .. code-block:: bash

        salt '*' xfs.prune_dump b74a3586-e52e-4a4a-8775-c3334fa8ea2c

    '''
    out = __salt__['cmd.run_all']("xfsinvutil -s {0} -F".format(sessionid))
    _verify_run(out)

    data = _xfs_prune_output(out['stdout'], sessionid)
    if data:
        return data

    raise CommandExecutionError("Session UUID \"{0}\" was not found.".format(sessionid))

Response: Prune the dump session identified by the given session id.


Instruction:
Can you generate the documentation for the following Python 3 function
def _xfs_estimate_output(out):
    '''
    Parse xfs_estimate output.
    '''
    spc = re.compile(r"\s+")
    data = {}
    for line in [l for l in out.split("\n") if l.strip()][1:]:
        directory, bsize, blocks, megabytes, logsize = spc.sub(" ", line).split(" ")
        data[directory] = {
            'block _size': bsize,
            'blocks': blocks,
            'megabytes': megabytes,
            'logsize': logsize,
        }

    return data

Response:
Parse xfs_estimate output.
    Parse xfs_estimate output.


Instruction:
Create a Python 3 function for
estimating the space that an XFS filesystem would take.

Response:
def estimate(path):
    '''
    Estimate the space that an XFS filesystem will take.
    For each directory estimate the space that directory would take
    if it were copied to an XFS filesystem.
    Estimation does not cross mount points.

    CLI Example:

    .. code-block:: bash

        salt '*' xfs.estimate /path/to/file
        salt '*' xfs.estimate /path/to/dir/*
    '''
    if not os.path.exists(path):
        raise CommandExecutionError("Path \"{0}\" was not found.".format(path))

    out = __salt__['cmd.run_all']("xfs_estimate -v {0}".format(path))
    _verify_run(out)

    return _xfs_estimate_output(out["stdout"])


Instruction:
How would you code a function in Python 3 to
create a new file system on the specified device.

Response:
def mkfs(device, label=None, ssize=None, noforce=None,
         bso=None, gmo=None, ino=None, lso=None, rso=None, nmo=None, dso=None):
    '''
    Create a file system on the specified device. By default wipes out with force.

    General options:

    * **label**: Specify volume label.
    * **ssize**: Specify the fundamental sector size of the filesystem.
    * **noforce**: Do not force create filesystem, if disk is already formatted.

    Filesystem geometry options:

    * **bso**: Block size options.
    * **gmo**: Global metadata options.
    * **dso**: Data section options. These options specify the location, size,
               and other parameters of the data section of the filesystem.
    * **ino**: Inode options to specify the inode size of the filesystem, and other inode allocation parameters.
    * **lso**: Log section options.
    * **nmo**: Naming options.
    * **rso**: Realtime section options.

    See the ``mkfs.xfs(8)`` manpage for a more complete description of corresponding options description.

    CLI Example:

    .. code-block:: bash

        salt '*' xfs.mkfs /dev/sda1
        salt '*' xfs.mkfs /dev/sda1 dso='su=32k,sw=6' noforce=True
        salt '*' xfs.mkfs /dev/sda1 dso='su=32k,sw=6' lso='logdev=/dev/sda2,size=10000b'
    '''

    getopts = lambda args: dict(((args and ("=" in args)
                                  and args or None)) and [kw.split("=") for kw in args.split(",")] or [])
    cmd = ["mkfs.xfs"]
    if label:
        cmd.append("-L")
        cmd.append("'{0}'".format(label))

    if ssize:
        cmd.append("-s")
        cmd.append(ssize)

    for switch, opts in [("-b", bso), ("-m", gmo), ("-n", nmo), ("-i", ino),
                         ("-d", dso), ("-l", lso), ("-r", rso)]:
        try:
            if getopts(opts):
                cmd.append(switch)
                cmd.append(opts)
        except Exception:
            raise CommandExecutionError("Wrong parameters \"{0}\" for option \"{1}\"".format(opts, switch))

    if not noforce:
        cmd.append("-f")
    cmd.append(device)

    cmd = ' '.join(cmd)
    out = __salt__['cmd.run_all'](cmd)
    _verify_run(out, cmd=cmd)

    return _parse_xfs_info(out['stdout'])


Instruction:
Given the following Python 3 function, write the documentation
def modify(device, label=None, lazy_counting=None, uuid=None):
    '''
    Modify parameters of an XFS filesystem.

    CLI Example:

    .. code-block:: bash

        salt '*' xfs.modify /dev/sda1 label='My backup' lazy_counting=False
        salt '*' xfs.modify /dev/sda1 uuid=False
        salt '*' xfs.modify /dev/sda1 uuid=True
    '''
    if not label and lazy_counting is None and uuid is None:
        raise CommandExecutionError("Nothing specified for modification for \"{0}\" device".format(device))

    cmd = ['xfs_admin']
    if label:
        cmd.append("-L")
        cmd.append("'{0}'".format(label))

    if lazy_counting is False:
        cmd.append("-c")
        cmd.append("0")
    elif lazy_counting:
        cmd.append("-c")
        cmd.append("1")

    if uuid is False:
        cmd.append("-U")
        cmd.append("nil")
    elif uuid:
        cmd.append("-U")
        cmd.append("generate")
    cmd.append(device)

    cmd = ' '.join(cmd)
    _verify_run(__salt__['cmd.run_all'](cmd), cmd=cmd)

    out = __salt__['cmd.run_all']("blkid -o export {0}".format(device))
    _verify_run(out)

    return _blkid_output(out['stdout'])

Response: Modify parameters of an XFS filesystem.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _get_mounts():
    '''
    List mounted filesystems.
    '''
    mounts = {}
    with salt.utils.files.fopen("/proc/mounts") as fhr:
        for line in salt.utils.data.decode(fhr.readlines()):
            device, mntpnt, fstype, options, fs_freq, fs_passno = line.strip().split(" ")
            if fstype != 'xfs':
                continue
            mounts[device] = {
                'mount_point': mntpnt,
                'options': options.split(","),
            }

    return mounts

Response: Return a list of mounted filesystems.


Instruction:
How would you explain what the following Python 3 function does
def defragment(device):
    '''
    Defragment mounted XFS filesystem.
    In order to mount a filesystem, device should be properly mounted and writable.

    CLI Example:

    .. code-block:: bash

        salt '*' xfs.defragment /dev/sda1
    '''
    if device == '/':
        raise CommandExecutionError("Root is not a device.")

    if not _get_mounts().get(device):
        raise CommandExecutionError("Device \"{0}\" is not mounted".format(device))

    out = __salt__['cmd.run_all']("xfs_fsr {0}".format(device))
    _verify_run(out)

    return {
        'log': out['stdout']
    }

Response: Defragment mounted XFS filesystem.


Instruction:
Create a Python 3 function to
return a checksum digest for a string

Response:
def digest(instr, checksum='md5'):
    '''
    Return a checksum digest for a string

    instr
        A string
    checksum : ``md5``
        The hashing algorithm to use to generate checksums. Valid options: md5,
        sha256, sha512.

    CLI Example:

    .. code-block:: bash

        salt '*' hashutil.digest 'get salted'
    '''
    hashing_funcs = {
        'md5': __salt__['hashutil.md5_digest'],
        'sha256': __salt__['hashutil.sha256_digest'],
        'sha512': __salt__['hashutil.sha512_digest'],
    }
    hash_func = hashing_funcs.get(checksum)

    if hash_func is None:
        raise salt.exceptions.CommandExecutionError(
                "Hash func '{0}' is not supported.".format(checksum))

    return hash_func(instr)


Instruction:
Can you generate the documentation for the following Python 3 function
def digest_file(infile, checksum='md5'):
    '''
    Return a checksum digest for a file

    infile
        A file path
    checksum : ``md5``
        The hashing algorithm to use to generate checksums. Wraps the
        :py:func:`hashutil.digest <salt.modules.hashutil.digest>` execution
        function.

    CLI Example:

    .. code-block:: bash

        salt '*' hashutil.digest_file /path/to/file
    '''
    if not __salt__['file.file_exists'](infile):
        raise salt.exceptions.CommandExecutionError(
                "File path '{0}' not found.".format(infile))

    with salt.utils.files.fopen(infile, 'rb') as f:
        file_hash = __salt__['hashutil.digest'](f.read(), checksum)

    return file_hash

Response: Returns a checksum digest for a file


Instruction:
Make a summary of the following Python 3 code
def base64_encodefile(fname):
    '''
    Read a file from the file system and return as a base64 encoded string

    .. versionadded:: 2016.3.0

    Pillar example:

    .. code-block:: yaml

        path:
          to:
            data: |
              {{ salt.hashutil.base64_encodefile('/path/to/binary_file') | indent(6) }}

    The :py:func:`file.decode <salt.states.file.decode>` state function can be
    used to decode this data and write it to disk.

    CLI Example:

    .. code-block:: bash

        salt '*' hashutil.base64_encodefile /path/to/binary_file
    '''
    encoded_f = BytesIO()

    with salt.utils.files.fopen(fname, 'rb') as f:
        base64.encode(f, encoded_f)

    encoded_f.seek(0)
    return salt.utils.stringutils.to_str(encoded_f.read())

Response: Read a file from the file system and return as a base64 encoded string


Instruction:
Can you tell what is the following Python 3 function doing
def hmac_signature(string, shared_secret, challenge_hmac):
    '''
    Verify a challenging hmac signature against a string / shared-secret

    .. versionadded:: 2014.7.0

    Returns a boolean if the verification succeeded or failed.

    CLI Example:

    .. code-block:: bash

        salt '*' hashutil.hmac_signature 'get salted' 'shared secret' 'eBWf9bstXg+NiP5AOwppB5HMvZiYMPzEM9W5YMm/AmQ='
    '''
    return salt.utils.hashutils.hmac_signature(string, shared_secret, challenge_hmac)

Response: Verify a challenging hmac signature against a string


Instruction:
Explain what the following Python 3 code does
def github_signature(string, shared_secret, challenge_hmac):
    '''
    Verify a challenging hmac signature against a string / shared-secret for
    github webhooks.

    .. versionadded:: 2017.7.0

    Returns a boolean if the verification succeeded or failed.

    CLI Example:

    .. code-block:: bash

        salt '*' hashutil.github_signature '{"ref":....} ' 'shared secret' 'sha1=bc6550fc290acf5b42283fa8deaf55cea0f8c206'
    '''
    msg = string
    key = shared_secret
    hashtype, challenge = challenge_hmac.split('=')
    if six.text_type:
        msg = salt.utils.stringutils.to_bytes(msg)
        key = salt.utils.stringutils.to_bytes(key)
    hmac_hash = hmac.new(key, msg, getattr(hashlib, hashtype))
    return hmac_hash.hexdigest() == challenge

Response: Verify a challenging hmac signature against a string


Instruction:
Here you have a function in Python 3, explain what it does
def cmd(
        name,
        fun=None,
        arg=(),
        **kwargs):
    '''
    Execute a runner asynchronous:

    USAGE:

    .. code-block:: yaml

        run_cloud:
          wheel.cmd:
            - fun: key.delete
            - match: minion_id
    '''
    ret = {'name': name,
           'changes': {},
           'comment': '',
           'result': True}
    if fun is None:
        fun = name
    client = salt.wheel.WheelClient(__opts__)
    low = {'fun': fun,
            'arg': arg,
            'kwargs': kwargs}
    client.cmd_async(low)
    return ret

Response: Execute a runner asynchronous for a specific key


Instruction:
Write a Python 3 script to
get inspectlib module for the lazy loader.

Response:
def _(module):
    '''
    Get inspectlib module for the lazy loader.

    :param module:
    :return:
    '''

    mod = None
    # pylint: disable=E0598
    try:
        # importlib is in Python 2.7+ and 3+
        import importlib
        mod = importlib.import_module("salt.modules.inspectlib.{0}".format(module))
    except ImportError:
        # No importlib around (2.6)
        mod = getattr(__import__("salt.modules.inspectlib", globals(), locals(), fromlist=[six.text_type(module)]), module)
    # pylint: enable=E0598

    mod.__grains__ = __grains__
    mod.__pillar__ = __pillar__
    mod.__salt__ = __salt__

    return mod


Instruction:
Can you tell what is the following Python 3 function doing
def inspect(mode='all', priority=19, **kwargs):
    '''
    Start node inspection and save the data to the database for further query.

    Parameters:

    * **mode**: Clarify inspection mode: configuration, payload, all (default)

      payload
        * **filter**: Comma-separated directories to track payload.

    * **priority**: (advanced) Set priority of the inspection. Default is low priority.

    CLI Example:

    .. code-block:: bash

        salt '*' inspector.inspect
        salt '*' inspector.inspect configuration
        salt '*' inspector.inspect payload filter=/opt,/ext/oracle
    '''
    collector = _("collector")
    try:
        return collector.Inspector(cachedir=__opts__['cachedir'],
                                   piddir=os.path.dirname(__opts__['pidfile']))\
            .request_snapshot(mode, priority=priority, **kwargs)
    except InspectorSnapshotException as ex:
        raise CommandExecutionError(ex)
    except Exception as ex:
        log.error(_get_error_message(ex))
        raise Exception(ex)

Response: Get the node information from the server and save it to the database.


Instruction:
Write a Python 3 script for
querying the node for specific information.

Response:
def query(*args, **kwargs):
    '''
    Query the node for specific information.

    Parameters:

    * **scope**: Specify scope of the query.

       * **System**: Return system data.

       * **Software**: Return software information.

       * **Services**: Return known services.

       * **Identity**: Return user accounts information for this system.
          accounts
            Can be either 'local', 'remote' or 'all' (equal to "local,remote").
            Remote accounts cannot be resolved on all systems, but only
            those, which supports 'passwd -S -a'.

          disabled
            True (or False, default) to return only disabled accounts.

       * **payload**: Payload scope parameters:
          filter
            Include only results which path starts from the filter string.

          time
            Display time in Unix ticks or format according to the configured TZ (default)
            Values: ticks, tz (default)

          size
            Format size. Values: B, KB, MB, GB

          type
            Include payload type.
            Values (comma-separated): directory (or dir), link, file (default)
            Example (returns everything): type=directory,link,file

          owners
            Resolve UID/GID to an actual names or leave them numeric (default).
            Values: name (default), id

          brief
            Return just a list of payload elements, if True. Default: False.

       * **all**: Return all information (default).

    CLI Example:

    .. code-block:: bash

        salt '*' inspector.query scope=system
        salt '*' inspector.query scope=payload type=file,link filter=/etc size=Kb brief=False
    '''
    query = _("query")
    try:
        return query.Query(kwargs.get('scope'), cachedir=__opts__['cachedir'])(*args, **kwargs)
    except InspectorQueryException as ex:
        raise CommandExecutionError(ex)
    except Exception as ex:
        log.error(_get_error_message(ex))
        raise Exception(ex)


Instruction:
How would you explain what the following Python 3 function does
def build(format='qcow2', path='/tmp/'):
    '''
    Build an image from a current system description.
    The image is a system image can be output in bootable ISO or QCOW2 formats.

    Node uses the image building library Kiwi to perform the actual build.

    Parameters:

    * **format**: Specifies output format: "qcow2" or "iso. Default: `qcow2`.
    * **path**: Specifies output path where to store built image. Default: `/tmp`.

    CLI Example:

    .. code-block:: bash

        salt myminion inspector.build
        salt myminion inspector.build format=iso path=/opt/builds/
    '''
    try:
        _("collector").Inspector(cachedir=__opts__['cachedir'],
                                 piddir=os.path.dirname(__opts__['pidfile']),
                                 pidfilename='').reuse_snapshot().build(format=format, path=path)
    except InspectorKiwiProcessorException as ex:
        raise CommandExecutionError(ex)
    except Exception as ex:
        log.error(_get_error_message(ex))
        raise Exception(ex)

Response: Build an image from a current system description.


Instruction:
Can you create a Python 3 function that
exports an image description for Kiwi.

Response:
def export(local=False, path="/tmp", format='qcow2'):
    '''
    Export an image description for Kiwi.

    Parameters:

    * **local**: Specifies True or False if the export has to be in the local file. Default: False.
    * **path**: If `local=True`, then specifies the path where file with the Kiwi description is written.
                Default: `/tmp`.

    CLI Example:

    .. code-block:: bash

        salt myminion inspector.export
        salt myminion inspector.export format=iso path=/opt/builds/
    '''
    if getpass.getuser() != 'root':
        raise CommandExecutionError('In order to export system, the minion should run as "root".')
    try:
        description = _("query").Query('all', cachedir=__opts__['cachedir'])()
        return _("collector").Inspector().reuse_snapshot().export(description, local=local, path=path, format=format)
    except InspectorKiwiProcessorException as ex:
        raise CommandExecutionError(ex)
    except Exception as ex:
        log.error(_get_error_message(ex))
        raise Exception(ex)


Instruction:
Write a Python 3 function that can
list current description snapshots.

Response:
def snapshots():
    '''
    List current description snapshots.

    CLI Example:

    .. code-block:: bash

        salt myminion inspector.snapshots
    '''
    try:
        return _("collector").Inspector(cachedir=__opts__['cachedir'],
                                        piddir=os.path.dirname(__opts__['pidfile'])).db.list()
    except InspectorSnapshotException as err:
        raise CommandExecutionError(err)
    except Exception as err:
        log.error(_get_error_message(err))
        raise Exception(err)


Instruction:
Can you implement a function in Python 3 that
deletes description snapshots from the system.

Response:
def delete(all=False, *databases):
    '''
    Remove description snapshots from the system.

    ::parameter: all. Default: False. Remove all snapshots, if set to True.

    CLI example:

    .. code-block:: bash

        salt myminion inspector.delete <ID> <ID1> <ID2>..
        salt myminion inspector.delete all=True
    '''
    if not all and not databases:
        raise CommandExecutionError('At least one database ID required.')

    try:
        ret = dict()
        inspector = _("collector").Inspector(cachedir=__opts__['cachedir'],
                                             piddir=os.path.dirname(__opts__['pidfile']))
        for dbid in all and inspector.db.list() or databases:
            ret[dbid] = inspector.db._db.purge(six.text_type(dbid))
        return ret
    except InspectorSnapshotException as err:
        raise CommandExecutionError(err)
    except Exception as err:
        log.error(_get_error_message(err))
        raise Exception(err)


Instruction:
Implement a function in Python 3 to
start a new instance of the NX - Log service.

Response:
def start(transport='zmq',
          address='0.0.0.0',
          port=49017,
          auth_address='0.0.0.0',
          auth_port=49018,
          disable_security=False,
          certificate=None,
          os_whitelist=None,
          os_blacklist=None,
          error_whitelist=None,
          error_blacklist=None,
          host_whitelist=None,
          host_blacklist=None):
    '''
    Listen to napalm-logs and publish events into the Salt event bus.

    transport: ``zmq``
        Choose the desired transport.

        .. note::
            Currently ``zmq`` is the only valid option.

    address: ``0.0.0.0``
        The address of the publisher, as configured on napalm-logs.

    port: ``49017``
        The port of the publisher, as configured on napalm-logs.

    auth_address: ``0.0.0.0``
        The address used for authentication
        when security is not disabled.

    auth_port: ``49018``
        Port used for authentication.

    disable_security: ``False``
        Trust unencrypted messages.
        Strongly discouraged in production.

    certificate: ``None``
        Absolute path to the SSL certificate.

    os_whitelist: ``None``
        List of operating systems allowed. By default everything is allowed.

    os_blacklist: ``None``
        List of operating system to be ignored. Nothing ignored by default.

    error_whitelist: ``None``
        List of errors allowed.

    error_blacklist: ``None``
        List of errors ignored.

    host_whitelist: ``None``
        List of hosts or IPs to be allowed.

    host_blacklist: ``None``
        List of hosts of IPs to be ignored.
    '''
    if not disable_security:
        if not certificate:
            log.critical('Please use a certificate, or disable the security.')
            return
        auth = napalm_logs.utils.ClientAuth(certificate,
                                            address=auth_address,
                                            port=auth_port)

    transport_recv_fun = _get_transport_recv(name=transport,
                                             address=address,
                                             port=port)
    if not transport_recv_fun:
        log.critical('Unable to start the engine', exc_info=True)
        return
    master = False
    if __opts__['__role'] == 'master':
        master = True
    while True:
        log.debug('Waiting for napalm-logs to send anything...')
        raw_object = transport_recv_fun()
        log.debug('Received from napalm-logs:')
        log.debug(raw_object)
        if not disable_security:
            dict_object = auth.decrypt(raw_object)
        else:
            dict_object = napalm_logs.utils.unserialize(raw_object)
        try:
            event_os = dict_object['os']
            if os_blacklist or os_whitelist:
                valid_os = salt.utils.stringutils.check_whitelist_blacklist(
                    event_os,
                    whitelist=os_whitelist,
                    blacklist=os_blacklist)
                if not valid_os:
                    log.info('Ignoring NOS %s as per whitelist/blacklist', event_os)
                    continue
            event_error = dict_object['error']
            if error_blacklist or error_whitelist:
                valid_error = salt.utils.stringutils.check_whitelist_blacklist(
                    event_error,
                    whitelist=error_whitelist,
                    blacklist=error_blacklist)
                if not valid_error:
                    log.info('Ignoring error %s as per whitelist/blacklist', event_error)
                    continue
            event_host = dict_object.get('host') or dict_object.get('ip')
            if host_blacklist or host_whitelist:
                valid_host = salt.utils.stringutils.check_whitelist_blacklist(
                    event_host,
                    whitelist=host_whitelist,
                    blacklist=host_blacklist)
                if not valid_host:
                    log.info('Ignoring messages from %s as per whitelist/blacklist', event_host)
                    continue
            tag = 'napalm/syslog/{os}/{error}/{host}'.format(
                os=event_os,
                error=event_error,
                host=event_host
            )
        except KeyError as kerr:
            log.warning('Missing keys from the napalm-logs object:', exc_info=True)
            log.warning(dict_object)
            continue  # jump to the next object in the queue
        log.debug('Sending event %s', tag)
        log.debug(raw_object)
        if master:
            event.get_master_event(__opts__,
                                   __opts__['sock_dir']
                                   ).fire_event(dict_object, tag)
        else:
            __salt__['event.send'](tag, dict_object)


Instruction:
Can you generate the documentation for the following Python 3 function
def configure_proxy(name, proxyname='p8000', start=True):
    '''
    Create the salt proxy file and start the proxy process
    if required

    Parameters:
        name:
            The name of this state
        proxyname:
            Name to be used for this proxy (should match entries in pillar)
        start:
            Boolean indicating if the process should be started

    Example:

    ..code-block:: yaml

        salt-proxy-configure:
            salt_proxy.configure_proxy:
                - proxyname: p8000
                - start: True

    '''
    ret = __salt__['salt_proxy.configure_proxy'](proxyname,
                                                 start=start)
    ret.update({
        'name': name,
        'comment': '{0} config messages'.format(name)
    })
    return ret

Response: Configure the salt proxy process


Instruction:
How would you explain what the following Python 3 function does
def _hadoop_cmd(module, command, *args):
    '''
       Hadoop/hdfs command wrapper

       As Hadoop command has been deprecated this module will default
       to use hdfs command and fall back to hadoop if it is not found

       In order to prevent random execution the module name is checked

       Follows hadoop command template:
          hadoop module -command args
       E.g.: hadoop dfs -ls /
    '''
    tool = 'hadoop'
    if salt.utils.path.which('hdfs'):
        tool = 'hdfs'

    out = None
    if module and command:
        if module in __authorized_modules__:
            mappings = {'tool': tool, 'module': module, 'command': command, 'args': ' '.join(args)}
            cmd = '{tool} {module} -{command} {args}'.format(**mappings)
            out = __salt__['cmd.run'](cmd, python_shell=False)
        else:
            return 'Error: Unknown module'
    else:
        return 'Error: Module and command not defined'
    return out

Response: Wrapper for hadoop command


Instruction:
Make a summary of the following Python 3 code
def dfs_present(path):
    '''
    Check if a file or directory is present on the distributed FS.

    CLI Example:

    .. code-block:: bash

        salt '*' hadoop.dfs_present /some_random_file

    Returns True if the file is present
    '''
    cmd_return = _hadoop_cmd('dfs', 'stat', path)
    match = 'No such file or directory'
    return False if match in cmd_return else True

Response: Check if a file or directory is present on the distributed FS.


Instruction:
Can you create a Python 3 function that
checks if a file or directory is absent on the distributed FS

Response:
def dfs_absent(path):
    '''
    Check if a file or directory is absent on the distributed FS.

    CLI Example:

    .. code-block:: bash

        salt '*' hadoop.dfs_absent /some_random_file

    Returns True if the file is absent
    '''
    cmd_return = _hadoop_cmd('dfs', 'stat', path)
    match = 'No such file or directory'
    return True if match in cmd_return else False


Instruction:
Create a Python 3 function for
waiting for subprocess to terminate and return subprocess

Response:
def run(self):
        '''
        wait for subprocess to terminate and return subprocess' return code.
        If timeout is reached, throw TimedProcTimeoutError
        '''
        def receive():
            if self.with_communicate:
                self.stdout, self.stderr = self.process.communicate(input=self.stdin)
            elif self.wait:
                self.process.wait()

        if not self.timeout:
            receive()
        else:
            rt = threading.Thread(target=receive)
            rt.start()
            rt.join(self.timeout)
            if rt.isAlive():
                # Subprocess cleanup (best effort)
                self.process.kill()

                def terminate():
                    if rt.isAlive():
                        self.process.terminate()
                threading.Timer(10, terminate).start()
                raise salt.exceptions.TimedProcTimeoutError(
                    '{0} : Timed out after {1} seconds'.format(
                        self.command,
                        six.text_type(self.timeout),
                    )
                )
        return self.process.returncode


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _get_full_stream(stream_name, region=None, key=None, keyid=None, profile=None):
    '''
    Get complete stream info from AWS, via describe_stream, including all shards.

    CLI example::

        salt myminion boto_kinesis._get_full_stream my_stream region=us-east-1
    '''
    conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile)
    r = {}
    stream = _get_basic_stream(stream_name, conn)['result']
    full_stream = stream

    # iterate through if there are > 100 shards (max that AWS will return from describe_stream)
    while stream["StreamDescription"]["HasMoreShards"]:
        stream = _execute_with_retries(conn,
                                       "describe_stream",
                                       StreamName=stream_name,
                                       ExclusiveStartShardId=stream["StreamDescription"]["Shards"][-1]["ShardId"])
        stream = stream['result']
        full_stream["StreamDescription"]["Shards"] += stream["StreamDescription"]["Shards"]

    r['result'] = full_stream
    return r

Response: Get complete stream info from AWS via describe_stream


Instruction:
Implement a Python 3 function for
getting the stream info from AWS returning only when the stream is in the ACTIVE state.

Response:
def get_stream_when_active(stream_name, region=None, key=None, keyid=None, profile=None):
    '''
    Get complete stream info from AWS, returning only when the stream is in the ACTIVE state.
    Continues to retry when stream is updating or creating.
    If the stream is deleted during retries, the loop will catch the error and break.

    CLI example::

        salt myminion boto_kinesis.get_stream_when_active my_stream region=us-east-1
    '''
    conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile)

    stream_status = None
    # only get basic stream until it's active,
    # so we don't pull the full list of shards repeatedly (in case of very large stream)
    attempt = 0
    max_retry_delay = 10
    while stream_status != "ACTIVE":
        time.sleep(_jittered_backoff(attempt, max_retry_delay))
        attempt += 1
        stream_response = _get_basic_stream(stream_name, conn)
        if 'error' in stream_response:
            return stream_response
        stream_status = stream_response['result']["StreamDescription"]["StreamStatus"]

    # now it's active, get the full stream if necessary
    if stream_response['result']["StreamDescription"]["HasMoreShards"]:
        stream_response = _get_full_stream(stream_name, region, key, keyid, profile)

    return stream_response


Instruction:
Here you have a function in Python 3, explain what it does
def exists(stream_name, region=None, key=None, keyid=None, profile=None):
    '''
    Check if the stream exists. Returns False and the error if it does not.

    CLI example::

        salt myminion boto_kinesis.exists my_stream region=us-east-1
    '''
    conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile)
    r = {}

    stream = _get_basic_stream(stream_name, conn)
    if 'error' in stream:
        r['result'] = False
        r['error'] = stream['error']
    else:
        r['result'] = True

    return r

Response: Check if the stream exists. Returns False and the error if it does not.


Instruction:
How would you code a function in Python 3 to
create a stream with name stream_name and initial number of shards num_shards.

Response:
def create_stream(stream_name, num_shards, region=None, key=None, keyid=None, profile=None):
    '''
    Create a stream with name stream_name and initial number of shards num_shards.

    CLI example::

        salt myminion boto_kinesis.create_stream my_stream N region=us-east-1
    '''
    conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile)
    r = _execute_with_retries(conn,
                              "create_stream",
                              ShardCount=num_shards,
                              StreamName=stream_name)
    if 'error' not in r:
        r['result'] = True
    return r


Instruction:
Create a Python 3 function for
deleting the stream with name stream_name. This cannot be undone! All data will be lost!!

Response:
def delete_stream(stream_name, region=None, key=None, keyid=None, profile=None):
    '''
    Delete the stream with name stream_name. This cannot be undone! All data will be lost!!

    CLI example::

        salt myminion boto_kinesis.delete_stream my_stream region=us-east-1
    '''
    conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile)
    r = _execute_with_retries(conn,
                              "delete_stream",
                              StreamName=stream_name)
    if 'error' not in r:
        r['result'] = True
    return r


Instruction:
Write a Python 3 script to
increase stream retention period to retention_hours

Response:
def increase_stream_retention_period(stream_name, retention_hours,
                                     region=None, key=None, keyid=None, profile=None):
    '''
    Increase stream retention period to retention_hours

    CLI example::

        salt myminion boto_kinesis.increase_stream_retention_period my_stream N region=us-east-1
    '''
    conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile)
    r = _execute_with_retries(conn,
                              "increase_stream_retention_period",
                              StreamName=stream_name,
                              RetentionPeriodHours=retention_hours)
    if 'error' not in r:
        r['result'] = True
    return r