Instruction:
Can you implement a function in Python 3 that
makes set efficient will convert all lists of items in levels to a set

Response:
def make_levels_set(levels):
    '''make set efficient will convert all lists of items
    in levels to a set to speed up operations'''
    for level_key,level_filters in levels.items():
        levels[level_key] = make_level_set(level_filters)
    return levels


Instruction:
Implement a function in Python 3 to
make level set will convert one level into a set

Response:
def make_level_set(level):
    '''make level set will convert one level into
    a set'''
    new_level = dict()
    for key,value in level.items():
        if isinstance(value,list):
            new_level[key] = set(value)
        else:
            new_level[key] = value
    return new_level


Instruction:
Create a Python 3 function for
extracting the file guts from an in memory tarfile.

Response:
def extract_guts(image_path,
                 tar,
                 file_filter=None,
                 tag_root=True,
                 include_sizes=True):

    '''extract the file guts from an in memory tarfile. The file is not closed.
       This should not be done for large images.
    '''
    if file_filter is None:
        file_filter = get_level('IDENTICAL')

    results = dict()
    digest = dict()
    allfiles = []

    if tag_root:
        roots = dict()

    if include_sizes:
        sizes = dict()

    for member in tar:
        member_name = member.name.replace('.','',1)
        allfiles.append(member_name)
        included = False
        if member.isdir() or member.issym():
            continue
        elif assess_content(member,file_filter):
            digest[member_name] = extract_content(image_path, member.name, return_hash=True)
            included = True
        elif include_file(member,file_filter):
            hasher = hashlib.md5()
            buf = member.tobuf()
            hasher.update(buf)
            digest[member_name] = hasher.hexdigest()
            included = True
        if included:
            if include_sizes:
                sizes[member_name] = member.size
            if tag_root:
                roots[member_name] = is_root_owned(member)

    results['all'] = allfiles
    results['hashes'] = digest
    if include_sizes:
        results['sizes'] = sizes
    if tag_root:
        results['root_owned'] = roots
    return results


Instruction:
Write a Python 3 function for
getting an in memory tar of an image

Response:
def get_memory_tar(image_path):
    '''get an in memory tar of an image. Use carefully, not as reliable
       as get_image_tar
    '''
    byte_array = Client.image.export(image_path)
    file_object = io.BytesIO(byte_array)
    tar = tarfile.open(mode="r|*", fileobj=file_object)
    return (file_object,tar)


Instruction:
Implement a Python 3 function for
getting an image tar either written in memory or to the file system.

Response:
def get_image_tar(image_path):
    '''get an image tar, either written in memory or to
    the file system. file_obj will either be the file object,
    or the file itself.
    '''
    bot.debug('Generate file system tar...')
    file_obj = Client.image.export(image_path=image_path)
    if file_obj is None:
        bot.error("Error generating tar, exiting.")
        sys.exit(1)
    tar = tarfile.open(file_obj)
    return file_obj, tar


Instruction:
Make a summary of the following Python 3 code
def delete_image_tar(file_obj, tar):
    '''delete image tar will close a file object (if extracted into
    memory) or delete from the file system (if saved to disk)'''
    try:
        file_obj.close()
    except:
        tar.close()
    if os.path.exists(file_obj):
        os.remove(file_obj)
        deleted = True
        bot.debug('Deleted temporary tar.')
    return deleted

Response: delete image tar will close a file object or delete from the file system


Instruction:
Given the following Python 3 function, write the documentation
def extract_content(image_path, member_name, return_hash=False):
    '''extract_content will extract content from an image using cat.
    If hash=True, a hash sum is returned instead
    '''
    if member_name.startswith('./'):
        member_name = member_name.replace('.','',1)
    if return_hash:
        hashy = hashlib.md5()

    try:
        content = Client.execute(image_path,'cat %s' %(member_name))
    except:
        return None

    if not isinstance(content,bytes):
        content = content.encode('utf-8')
        content = bytes(content)

    # If permissions don't allow read, return None
    if len(content) == 0:
        return None
    if return_hash:
        hashy.update(content)
        return hashy.hexdigest()
    return content

Response: extract_content will extract content from an image using cat.


Instruction:
Explain what the following Python 3 code does
def run_build(build_dir, params, verbose=True):
    '''run_build takes a build directory and params dictionary, and does the following:
      - downloads repo to a temporary directory
      - changes branch or commit, if needed
      - creates and bootstraps singularity image from Singularity file
      - returns a dictionary with:
          image (path), metadata (dict)

    The following must be included in params:
       spec_file, repo_url, branch, commit

    '''

    # Download the repository

    download_repo(repo_url=params['repo_url'],
                  destination=build_dir)

    os.chdir(build_dir)

    if params['branch'] != None:
        bot.info('Checking out branch %s' %params['branch'])
        os.system('git checkout %s' %(params['branch']))
    else:
        params['branch'] = "master"

    # Set the debug level

    Client.debug = params['debug']

    # Commit

    if params['commit'] not in [None,'']:
        bot.info('Checking out commit %s' %params['commit'])
        os.system('git checkout %s .' %(params['commit']))

    # From here on out commit is used as a unique id, if we don't have one, we use current
    else:
        params['commit'] = os.popen('git log -n 1 --pretty=format:"%H"').read()
        bot.warning("commit not specified, setting to current %s" %params['commit'])

    # Dump some params for the builder, in case it fails after this
    passing_params = "/tmp/params.pkl"
    pickle.dump(params, open(passing_params,'wb'))

    # Now look for spec file
    if os.path.exists(params['spec_file']):
        bot.info("Found spec file %s in repository" %params['spec_file'])

        # If the user has a symbolic link
        if os.path.islink(params['spec_file']):
            bot.info("%s is a symbolic link." %params['spec_file'])
            params['spec_file'] = os.path.realpath(params['spec_file'])

        # START TIMING
        start_time = datetime.now()

        # Secure Build
        image = Client.build(recipe=params['spec_file'],
                             build_folder=build_dir,
                             isolated=True)

        # Save has for metadata (also is image name)
        version = get_image_file_hash(image)
        params['version'] = version
        pickle.dump(params, open(passing_params,'wb'))

        # Rename image to be hash
        finished_image = "%s/%s.simg" %(os.path.dirname(image), version)
        image = shutil.move(image, finished_image)

        final_time = (datetime.now() - start_time).seconds
        bot.info("Final time of build %s seconds." %final_time)

        # Did the container build successfully?
        test_result = test_container(image)
        if test_result['return_code'] != 0:
            bot.error("Image failed to build, cancelling.")
            sys.exit(1)

        # Get singularity version
        singularity_version = Client.version()
        Client.debug = False
        inspect = Client.inspect(image) # this is a string
        Client.debug = params['debug']

        # Get information on apps
        Client.debug = False
        app_names = Client.apps(image)
        Client.debug = params['debug']
        apps = extract_apps(image, app_names)

        metrics = {'build_time_seconds': final_time,
                   'singularity_version': singularity_version,
                   'singularity_python_version': singularity_python_version,
                   'inspect': inspect,
                   'version': version,
                   'apps': apps}

        output = {'image':image,
                  'metadata':metrics,
                  'params':params }

        return output

    else:

        # Tell the user what is actually there
        present_files = glob("*")
        bot.error("Build file %s not found in repository" %params['spec_file'])
        bot.info("Found files are %s" %"\n".join(present_files))
        # Params have been exported, will be found by log
        sys.exit(1)

Response: This function downloads the repository and builds the singularity image from Singularity and creates the singularity image from Singularity.


Instruction:
Explain what the following Python 3 code does
def send_build_data(build_dir, data, secret,
                    response_url=None,clean_up=True):
    '''finish build sends the build and data (response) to a response url
    :param build_dir: the directory of the build
    :response_url: where to send the response. If None, won't send
    :param data: the data object to send as a post
    :param clean_up: If true (default) removes build directory
    '''
    # Send with Authentication header
    body = '%s|%s|%s|%s|%s' %(data['container_id'],
                              data['commit'],
                              data['branch'],
                              data['token'],
                              data['tag'])

    signature = generate_header_signature(secret=secret,
                                          payload=body,
                                          request_type="push")

    headers = {'Authorization': signature }

    if response_url is not None:
        finish = requests.post(response_url,data=data, headers=headers)
        bot.debug("RECEIVE POST TO SINGULARITY HUB ---------------------")
        bot.debug(finish.status_code)
        bot.debug(finish.reason)
    else:
        bot.warning("response_url set to None, skipping sending of build.")

    if clean_up == True:
        shutil.rmtree(build_dir)

    # Delay a bit, to give buffer between bringing instance down
    time.sleep(20)

Response: finish build sends the build and data to a response url


Instruction:
Explain what the following Python 3 code does
def send_build_close(params,response_url):
    '''send build close sends a final response (post) to the server to bring down
    the instance. The following must be included in params:

    repo_url, logfile, repo_id, secret, log_file, token
    '''
    # Finally, package everything to send back to shub
    response = {"log": json.dumps(params['log_file']),
                "repo_url": params['repo_url'],
                "logfile": params['logfile'],
                "repo_id": params['repo_id'],
                "container_id": params['container_id']}

    body = '%s|%s|%s|%s|%s' %(params['container_id'],
                              params['commit'],
                              params['branch'],
                              params['token'],
                              params['tag'])

    signature = generate_header_signature(secret=params['token'],
                                          payload=body,
                                          request_type="finish")

    headers = {'Authorization': signature }

    finish = requests.post(response_url,data=response, headers=headers)
    bot.debug("FINISH POST TO SINGULARITY HUB ---------------------")
    bot.debug(finish.status_code)
    bot.debug(finish.reason)
    return finish

Response: send build close to the server


Instruction:
Can you generate a brief explanation for the following Python 3 code
def remove_unicode_dict(input_dict):
    '''remove unicode keys and values from dict, encoding in utf8
    '''
    if isinstance(input_dict, collections.Mapping):
        return dict(map(remove_unicode_dict, input_dict.iteritems()))
    elif isinstance(input_dict, collections.Iterable):
        return type(input_dict)(map(remove_unicode_dict, input_dict))
    else:
        return input_dict

Response: remove unicode keys and values from dict encoding in utf8


Instruction:
Can you generate a brief explanation for the following Python 3 code
def update_dict(input_dict,key,value):
    '''update_dict will update lists in a dictionary. If the key is not included,
    if will add as new list. If it is, it will append.
    :param input_dict: the dict to update
    :param value: the value to update with
    '''
    if key in input_dict:
        input_dict[key].append(value)
    else:
        input_dict[key] = [value]
    return input_dict

Response: update_dict will update lists in a dictionary.


Instruction:
Make a summary of the following Python 3 code
def get_google_service(service_type=None,version=None):
    '''
    get_url will use the requests library to get a url
    :param service_type: the service to get (default is storage)
    :param version: version to use (default is v1)
    '''
    if service_type == None:
        service_type = "storage"
    if version == None:
        version = "v1"

    credentials = GoogleCredentials.get_application_default()
    return build(service_type, version, credentials=credentials)

Response: get_url will use the requests library to get a url


Instruction:
Make a summary of the following Python 3 code
def upload_file(storage_service,bucket,bucket_path,file_name,verbose=True):
    '''get_folder will return the folder with folder_name, and if create=True,
    will create it if not found. If folder is found or created, the metadata is
    returned, otherwise None is returned
    :param storage_service: the drive_service created from get_storage_service
    :param bucket: the bucket object from get_bucket
    :param file_name: the name of the file to upload
    :param bucket_path: the path to upload to
    '''
    # Set up path on bucket
    upload_path = "%s/%s" %(bucket['id'],bucket_path)
    if upload_path[-1] != '/':
        upload_path = "%s/" %(upload_path)
    upload_path = "%s%s" %(upload_path,os.path.basename(file_name))
    body = {'name': upload_path }
    # Create media object with correct mimetype
    if os.path.exists(file_name):
        mimetype = sniff_extension(file_name,verbose=verbose)
        media = http.MediaFileUpload(file_name,
                                     mimetype=mimetype,
                                     resumable=True)
        request = storage_service.objects().insert(bucket=bucket['id'],
                                                   body=body,
                                                   predefinedAcl="publicRead",
                                                   media_body=media)
        result = request.execute()
        return result
    bot.warning('%s requested for upload does not exist, skipping' %file_name)

Response: upload a file to the specified bucket


Instruction:
Can you generate the documentation for the following Python 3 function
def get_image_path(repo_url, trailing_path):
    '''get_image_path will determine an image path based on a repo url, removing
    any token, and taking into account urls that end with .git.
    :param repo_url: the repo url to parse:
    :param trailing_path: the trailing path (commit then hash is common)
    '''
    repo_url = repo_url.split('@')[-1].strip()
    if repo_url.endswith('.git'):
        repo_url =  repo_url[:-4]
    return "%s/%s" %(re.sub('^http.+//www[.]','',repo_url), trailing_path)

Response: This function will determine the path to the image file based on a repo url and the trailing path.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def run_build(logfile='/tmp/.shub-log'):

    '''run_build will generate the Singularity build from a spec_file from a repo_url.

    If no arguments are required, the metadata api is queried for the values.

    :param build_dir: directory to do the build in. If not specified, will use temporary.
    :param spec_file: the spec_file name to use, assumed to be in git repo
    :param repo_url: the url to download the repo from
    :param repo_id: the repo_id to uniquely identify the repo (in case name changes)
    :param commit: the commit to checkout. If none provided, will use most recent.
    :param bucket_name: the name of the bucket to send files to
    :param verbose: print out extra details as we go (default True)
    :param token: a token to send back to the server to authenticate the collection
    :param secret: a secret to match to the correct container
    :param response_url: the build url to send the response back to. Should also come
    from metadata. If not specified, no response is sent
    :param branch: the branch to checkout for the build.

    :: note: this function is currently configured to work with Google Compute
    Engine metadata api, and should (will) be customized if needed to work elsewhere

    '''

    # If we are building the image, this will not be set
    go = get_build_metadata(key='dobuild')
    if go == None:
        sys.exit(0)

    # If the user wants debug, this will be set
    debug = True
    enable_debug = get_build_metadata(key='debug')
    if enable_debug == None:
        debug = False
    bot.info('DEBUG %s' %debug)

    # Uaw /tmp for build directory
    build_dir = tempfile.mkdtemp()

    # Get variables from the instance metadata API
    metadata = [{'key': 'repo_url', 'value': None },
                {'key': 'repo_id', 'value': None },
                {'key': 'response_url', 'value': None },
                {'key': 'bucket_name', 'value': None },
                {'key': 'tag', 'value': None },
                {'key': 'container_id', 'value': None },
                {'key': 'commit', 'value': None },
                {'key': 'token', 'value': None},
                {'key': 'branch', 'value': None },
                {'key': 'spec_file', 'value': None},
                {'key': 'logging_url', 'value': None },
                {'key': 'logfile', 'value': logfile }]

    # Obtain values from build
    bot.log("BUILD PARAMETERS:")
    params = get_build_params(metadata)
    params['debug'] = debug

    # Default spec file is Singularity
    if params['spec_file'] == None:
        params['spec_file'] = "Singularity"

    if params['bucket_name'] == None:
        params['bucket_name'] = "singularityhub"

    if params['tag'] == None:
        params['tag'] = "latest"

    output = run_build_main(build_dir=build_dir,
                            params=params)

    # Output includes:
    finished_image = output['image']
    metadata = output['metadata']
    params = output['params']

    # Upload image package files to Google Storage
    if os.path.exists(finished_image):
        bot.info("%s successfully built" %finished_image)
        dest_dir = tempfile.mkdtemp(prefix='build')

        # The path to the images on google drive will be the github url/commit folder
        trailing_path = "%s/%s" %(params['commit'], params['version'])
        image_path = get_image_path(params['repo_url'], trailing_path)
                                                         # commits are no longer unique
                                                         # storage is by commit

        build_files = [finished_image]
        bot.info("Sending image to storage:")
        bot.info('\n'.join(build_files))

        # Start the storage service, retrieve the bucket
        storage_service = get_google_service() # default is "storage" "v1"
        bucket = get_bucket(storage_service,params["bucket_name"])

        # For each file, upload to storage
        files = []
        for build_file in build_files:
            bot.info("Uploading %s to storage..." %build_file)
            storage_file = upload_file(storage_service,
                                       bucket=bucket,
                                       bucket_path=image_path,
                                       file_name=build_file)
            files.append(storage_file)

        # Finally, package everything to send back to shub
        response = {"files": json.dumps(files),
                    "repo_url": params['repo_url'],
                    "commit": params['commit'],
                    "repo_id": params['repo_id'],
                    "branch": params['branch'],
                    "tag": params['tag'],
                    "container_id": params['container_id'],
                    "spec_file":params['spec_file'],
                    "token": params['token'],
                    "metadata": json.dumps(metadata)}

        # Did the user specify a specific log file?
        custom_logfile = get_build_metadata('logfile')
        if custom_logfile is not None:
            logfile = custom_logfile
        response['logfile'] = logfile

        # Send final build data to instance
        send_build_data(build_dir=build_dir,
                        response_url=params['response_url'],
                        secret=params['token'],
                        data=response)

        # Dump final params, for logger to retrieve
        passing_params = "/tmp/params.pkl"
        pickle.dump(params,open(passing_params,'wb'))

Response: This function will generate the Singularity build from a spec_file from a repo_url and then run it.


Instruction:
Make a summary of the following Python 3 code
def get_build_metadata(key):
    '''get_build_metadata will return metadata about an instance from within it.
    :param key: the key to look up
    '''
    headers = {"Metadata-Flavor":"Google"}
    url = "http://metadata.google.internal/computeMetadata/v1/instance/attributes/%s" %(key)
    response = requests.get(url=url,headers=headers)
    if response.status_code == 200:
        return response.text
    return None

Response: get_build_metadata will return metadata about an instance from within it


Instruction:
Can you tell what is the following Python 3 function doing
def get_build_params(metadata):
    '''get_build_params uses get_build_metadata to retrieve corresponding meta data values for a build
    :param metadata: a list, each item a dictionary of metadata, in format:
    metadata = [{'key': 'repo_url', 'value': repo_url },
                {'key': 'repo_id', 'value': repo_id },
                {'key': 'credential', 'value': credential },
                {'key': 'response_url', 'value': response_url },
                {'key': 'token', 'value': token},
                {'key': 'commit', 'value': commit }]

    '''
    params = dict()
    for item in metadata:
        if item['value'] == None:
            response = get_build_metadata(key=item['key'])
            item['value'] = response
        params[item['key']] = item['value']
        if item['key'] not in ['token', 'secret', 'credential']:
            bot.info('%s is set to %s' %(item['key'],item['value']))
    return params

Response: uses get_build_metadata to retrieve corresponding meta data values for a build


Instruction:
Explain what the following Python 3 code does
def rsync(*args, **kwargs):
    kwargs.setdefault('capture', False)
    replacements = dict(
        host_string="{user}@{host}".format(
            user=env.instance.config.get('user', 'root'),
            host=env.instance.config.get(
                'host', env.instance.config.get(
                    'ip', env.instance.uid))))
    args = [x.format(**replacements) for x in args]
    ssh_info = env.instance.init_ssh_key()
    ssh_info.pop('host')
    ssh_info.pop('user')
    ssh_args = env.instance.ssh_args_from_info(ssh_info)
    cmd_parts = ['rsync']
    cmd_parts.extend(['-e', "ssh %s" % shjoin(ssh_args)])
    cmd_parts.extend(args)
    cmd = shjoin(cmd_parts)
    return local(cmd, **kwargs)

Response: wrapper around the rsync command.


Instruction:
Implement a Python 3 function for
bootstrapping an EC2 instance that has been booted into an AMI from http://www. daemonology. net

Response:
def bootstrap(**kwargs):
    """ Bootstrap an EC2 instance that has been booted into an AMI from http://www.daemonology.net/freebsd-on-ec2/
    Note: deprecated, current AMI images are basically pre-bootstrapped, they just need to be configured.
    """
    # the user for the image is `ec2-user`, there is no sudo, but we can su to root w/o password
    original_host = env.host_string
    env.host_string = 'ec2-user@%s' % env.instance.uid
    bootstrap_files = env.instance.config.get('bootstrap-files', 'bootstrap-files')
    put('%s/authorized_keys' % bootstrap_files, '/tmp/authorized_keys')
    put(join(bsdploy_path, 'enable_root_login_on_daemonology.sh'), '/tmp/', mode='0775')
    run("""su root -c '/tmp/enable_root_login_on_daemonology.sh'""")
    # revert back to root
    env.host_string = original_host
    # give sshd a chance to restart
    sleep(2)
    run('rm /tmp/enable_root_login_on_daemonology.sh')

    # allow overwrites from the commandline
    env.instance.config.update(kwargs)

    bu = BootstrapUtils()
    bu.ssh_keys = None
    bu.upload_authorized_keys = False
    bu.bootstrap_files_yaml = 'daemonology-files.yml'
    bu.print_bootstrap_files()

    bu.create_bootstrap_directories()
    bu.upload_bootstrap_files({})
    # we need to install python here, because there is no way to install it via
    # ansible playbooks
    bu.install_pkg('/', chroot=False, packages=['python27'])


Instruction:
Write a Python 3 script for
bootstrapping the FreeBSD droplet into a new root.

Response:
def bootstrap(**kwargs):
    """Digital Oceans FreeBSD droplets are pretty much already pre-bootstrapped,
    including having python2.7 and sudo etc. pre-installed.
    the only thing we need to change is to allow root to login (without a password)
    enable pf and ensure it is running
    """

    bu = BootstrapUtils()
    # (temporarily) set the user to `freebsd`
    original_host = env.host_string
    env.host_string = 'freebsd@%s' % env.instance.uid
    # copy DO bsdclout-init results:
    if bu.os_release.startswith('10'):
        sudo("""cat /etc/rc.digitalocean.d/droplet.conf > /etc/rc.conf""")
    sudo("""sysrc zfs_enable=YES""")
    sudo("""sysrc sshd_enable=YES""")
    # enable and start pf
    sudo("""sysrc pf_enable=YES""")
    sudo("""sysrc -f /boot/loader.conf pfload=YES""")
    sudo('kldload pf', warn_only=True)
    sudo('''echo 'pass in all' > /etc/pf.conf''')
    sudo('''echo 'pass out all' >> /etc/pf.conf''')
    sudo('''chmod 644 /etc/pf.conf''')
    sudo('service pf start')
    # overwrite sshd_config, because the DO version only contains defaults
    # and a line explicitly forbidding root to log in
    sudo("""echo 'PermitRootLogin without-password' > /etc/ssh/sshd_config""")
    # additionally, make sure the root user is unlocked!
    sudo('pw unlock root')
    # overwrite the authorized keys for root, because DO creates its entries to explicitly
    # disallow root login
    bootstrap_files = env.instance.config.get('bootstrap-files', 'bootstrap-files')
    put(path.abspath(path.join(env['config_base'], bootstrap_files, 'authorized_keys')), '/tmp/authorized_keys', use_sudo=True)
    sudo('''mv /tmp/authorized_keys /root/.ssh/''')
    sudo('''chown root:wheel /root/.ssh/authorized_keys''')

    sudo("""service sshd fastreload""")
    # revert back to root
    env.host_string = original_host
    # give sshd a chance to restart
    sleep(2)
    # clean up DO cloudinit leftovers
    run("rm -f /etc/rc.d/digitalocean")
    run("rm -rf /etc/rc.digitalocean.d")
    run("rm -rf /usr/local/bsd-cloudinit/")
    run("pkg remove -y avahi-autoipd || true")

    # allow overwrites from the commandline
    env.instance.config.update(kwargs)

    bu.ssh_keys = None
    bu.upload_authorized_keys = False


Instruction:
How would you explain what the following Python 3 function does
def bootstrap_files(self):
        bootstrap_file_yamls = [
            abspath(join(self.default_template_path, self.bootstrap_files_yaml)),
            abspath(join(self.custom_template_path, self.bootstrap_files_yaml))]
        bootstrap_files = dict()
        if self.upload_authorized_keys:
            bootstrap_files['authorized_keys'] = BootstrapFile(self, 'authorized_keys', **{
                'directory': '/mnt/root/.ssh',
                'directory_mode': '0600',
                'remote': '/mnt/root/.ssh/authorized_keys',
                'fallback': [
                    '~/.ssh/identity.pub',
                    '~/.ssh/id_rsa.pub',
                    '~/.ssh/id_dsa.pub',
                    '~/.ssh/id_ecdsa.pub']})
        for bootstrap_file_yaml in bootstrap_file_yamls:
            if not exists(bootstrap_file_yaml):
                continue
            with open(bootstrap_file_yaml) as f:
                info = yaml.load(f, Loader=SafeLoader)
            if info is None:
                continue
            for k, v in info.items():
                bootstrap_files[k] = BootstrapFile(self, k, **v)

        for bf in bootstrap_files.values():
            if not exists(bf.local) and bf.raw_fallback:
                if not bf.existing_fallback:
                    print("Found no public key in %s, you have to create '%s' manually" % (expanduser('~/.ssh'), bf.local))
                    sys.exit(1)
                print("The '%s' file is missing." % bf.local)
                for path in bf.existing_fallback:
                    yes = env.instance.config.get('bootstrap-yes', False)
                    if yes or yesno("Should we generate it using the key in '%s'?" % path):
                        if not exists(bf.expected_path):
                            os.mkdir(bf.expected_path)
                        with open(bf.local, 'wb') as out:
                            with open(path, 'rb') as f:
                                out.write(f.read())
                        break
                else:
                    # answered no to all options
                    sys.exit(1)

            if not bf.check():
                print('Cannot find %s' % bf.local)
                sys.exit(1)

        packages_path = join(self.download_path, 'packages')
        if exists(packages_path):
            for dirpath, dirnames, filenames in os.walk(packages_path):
                path = dirpath.split(packages_path)[1][1:]
                for filename in filenames:
                    if not filename.endswith('.txz'):
                        continue
                    bootstrap_files[join(path, filename)] = BootstrapFile(
                        self, join(path, filename), **dict(
                            local=join(packages_path, join(path, filename)),
                            remote=join('/mnt/var/cache/pkg/All', filename),
                            encrypted=False))

        if self.ssh_keys is not None:
            for ssh_key_name, ssh_key_options in list(self.ssh_keys):
                ssh_key = join(self.custom_template_path, ssh_key_name)
                if exists(ssh_key):
                    pub_key_name = '%s.pub' % ssh_key_name
                    pub_key = '%s.pub' % ssh_key
                    if not exists(pub_key):
                        print("Public key '%s' for '%s' missing." % (pub_key, ssh_key))
                        sys.exit(1)
                    bootstrap_files[ssh_key_name] = BootstrapFile(
                        self, ssh_key_name, **dict(
                            local=ssh_key,
                            remote='/mnt/etc/ssh/%s' % ssh_key_name,
                            mode=0600))
                    bootstrap_files[pub_key_name] = BootstrapFile(
                        self, pub_key_name, **dict(
                            local=pub_key,
                            remote='/mnt/etc/ssh/%s' % pub_key_name,
                            mode=0644))
        if hasattr(env.instance, 'get_vault_lib'):
            vaultlib = env.instance.get_vault_lib()
            for bf in bootstrap_files.values():
                if bf.encrypted is None and exists(bf.local):
                    with open(bf.local) as f:
                        data = f.read()
                    bf.info['encrypted'] = vaultlib.is_encrypted(data)
        return bootstrap_files

Response: This method will create a bootstrap file for the FreeBSD installation.


Instruction:
Can you write a function in Python 3 where it
returns the name of the disk devices that are suitable installation targets by subtracting CDROM - and USB devices from the list of total mounts.

Response:
def devices(self):
        """ computes the name of the disk devices that are suitable
        installation targets by subtracting CDROM- and USB devices
        from the list of total mounts.
        """
        install_devices = self.install_devices
        if 'bootstrap-system-devices' in env.instance.config:
            devices = set(env.instance.config['bootstrap-system-devices'].split())
        else:
            devices = set(self.sysctl_devices)
            for sysctl_device in self.sysctl_devices:
                for install_device in install_devices:
                    if install_device.startswith(sysctl_device):
                        devices.remove(sysctl_device)
        return devices


Instruction:
How would you code a function in Python 3 to
download bootstrap assets to control host.

Response:
def fetch_assets(self):
        """ download bootstrap assets to control host.
        If present on the control host they will be uploaded to the target host during bootstrapping.
        """
        # allow overwrites from the commandline
        packages = set(
            env.instance.config.get('bootstrap-packages', '').split())
        packages.update(['python27'])
        cmd = env.instance.config.get('bootstrap-local-download-cmd', 'wget -c -O "{0.local}" "{0.url}"')
        items = sorted(self.bootstrap_files.items())
        for filename, asset in items:
            if asset.url:
                if not exists(dirname(asset.local)):
                    os.makedirs(dirname(asset.local))
                local(cmd.format(asset))
            if filename == 'packagesite.txz':
                # add packages to download
                items.extend(self._fetch_packages(asset.local, packages))


Instruction:
Write a Python 3 script to
return a string representation of the given response object.

Response:
def res_to_str(res):
    """
    :param res: :class:`requests.Response` object

    Parse the given request and generate an informative string from it
    """
    if 'Authorization' in res.request.headers:
        res.request.headers['Authorization'] = "*****"
    return """
####################################
url = %s
headers = %s
-------- data sent -----------------
%s
------------------------------------
@@@@@ response @@@@@@@@@@@@@@@@
headers = %s
code = %d
reason = %s
--------- data received ------------
%s
------------------------------------
####################################
""" % (res.url,
       str(res.request.headers),
       OLD_REQ and res.request.data or res.request.body,
       res.headers,
       res.status_code,
       res.reason,
       res.text)


Instruction:
Create a Python 3 function to
parse a resource definition and return all the info extracted from the resource section of the apipie

Response:
def parse_resource_definition(resource_name, resource_dct):
    """
    Returns all the info extracted from a resource section of the apipie json

    :param resource_name: Name of the resource that is defined by the section
    :param resrouce_dict: Dictionary as generated by apipie of the resource
        definition
    """
    new_dict = {
        '__module__': resource_dct.get('__module__', __name__),
        '__doc__': resource_dct['full_description'],
        '_resource_name': resource_name,
        '_own_methods': set(),
        '_conflicting_methods': [],
    }

    # methods in foreign_methods are meant for other resources,
    # that is, the url and the resource field do not match /api/{resource}
    foreign_methods = {}

    # as defined per apipie gem, each method can have more than one api,
    # for example, /api/hosts can have the GET /api/hosts api and the GET
    # /api/hosts/:id api or DELETE /api/hosts
    for method in resource_dct['methods']:
        # set the docstring if it only has one api
        if not new_dict['__doc__'] and len(method['apis']) == 1:
            new_dict['__doc__'] = \
                method['apis'][0]['short_description']
        for api in method['apis']:
            api = MethodAPIDescription(resource_name, method, api)

            if api.resource != resource_name:
                # this means that the json apipie passed says that an
                # endpoint in the form: /api/{resource}/* belongs to
                # {different_resource}, we just put it under {resource}
                # later, storing it under _foreign_methods for now as we
                # might not have parsed {resource} yet
                functions = foreign_methods.setdefault(api.resource, {})
                if api.name in functions:
                    old_api = functions.get(api.name).defs
                    # show only in debug the repeated but identical definitions
                    log_method = logger.warning
                    if api.url == old_api.url:
                        log_method = logger.debug

                    log_method(
                        "There is a conflict trying to redefine a method "
                        "for a foreign resource (%s): \n"
                        "\tresource:\n"
                        "\tapipie_resource: %s\n"
                        "\tnew_api: %s\n"
                        "\tnew_url: %s\n"
                        "\told_api: %s\n"
                        "\told_url: %s",
                        api.name,
                        resource_name,
                        pprint.pformat(api),
                        api.url,
                        pprint.pformat(old_api),
                        old_api.url,
                    )
                    new_dict['_conflicting_methods'].append(api)
                    continue
                functions[api.name] = api.generate_func()

            else:
                # it's an own method, resource and url match
                if api.name in new_dict['_own_methods']:
                    old_api = new_dict.get(api.name).defs
                    log_method = logger.warning
                    # show only in debug the repeated but identical definitions
                    if api.url == old_api.url:
                        log_method = logger.debug

                    log_method(
                        "There is a conflict trying to redefine method "
                        "(%s): \n"
                        "\tapipie_resource: %s\n"
                        "\tnew_api: %s\n"
                        "\tnew_url: %s\n"
                        "\told_api: %s\n"
                        "\told_url: %s",
                        api.name,
                        resource_name,
                        pprint.pformat(api),
                        api.url,
                        pprint.pformat(old_api),
                        old_api.url,
                    )
                    new_dict['_conflicting_methods'].append(api)
                    continue
                new_dict['_own_methods'].add(api.name)
                new_dict[api.name] = api.generate_func()

    return new_dict, foreign_methods


Instruction:
Can you implement a function in Python 3 that
parses the resource name from the given URL.

Response:
def parse_resource_from_url(self, url):
        """
        Returns the appropriate resource name for the given URL.

        :param url:  API URL stub, like: '/api/hosts'
        :return: Resource name, like 'hosts', or None if not found
        """
        # special case for the api root
        if url == '/api':
            return 'api'
        elif url == '/katello':
            return 'katello'

        match = self.resource_pattern.match(url)
        if match:
            return match.groupdict().get('resource', None)


Instruction:
Given the following Python 3 function, write the documentation
def _get_name(self):
        if self.url.count(':') > 1:
            # /api/one/two/:three/four -> two_:three_four
            base_name = self.url.split('/', 3)[-1].replace('/', '_')[1:]
            # :one_two_three -> two_three
            if base_name.startswith(':'):
                base_name = base_name.split('_')[-1]
            # one_:two_three_:four_five -> one_three_five
            base_name = re.sub('_:[^/]+', '', base_name)
            # in case that the last term was a parameter
            if base_name.endswith('_'):
                base_name = base_name[:-1]
            # one_two_three -> one_two_method_three
            base_name = (
                '_' + self._method['name']
            ).join(base_name.rsplit('_', 1))
        else:
            base_name = self._method['name']
        if base_name == 'import':
            base_name = 'import_'
        if self._apipie_resource != self.resource:
            return '%s_%s' % (self._apipie_resource, base_name)
        else:
            return base_name

Response: Get the name of the current object.


Instruction:
How would you explain what the following Python 3 function does
def generate_func(self, as_global=False):
        keywords = []
        params_def = []
        params_doc = ""
        original_names = {}

        params = dict(
            (param['name'], param)
            for param in self.params
        )

        # parse the url required params, as sometimes they are skipped in the
        # parameters list of the definition
        for param in self.url_params:
            if param not in params:
                param = {
                    'name': param,
                    'required': True,
                    'description': '',
                    'validator': '',
                }
                params[param['name']] = param
            else:
                params[param]['required'] = True

        # split required and non-required params for the definition
        req_params = []
        nonreq_params = []
        for param in six.itervalues(params):
            if param['required']:
                req_params.append(param)
            else:
                nonreq_params.append(param)

        for param in req_params + nonreq_params:
            params_doc += self.create_param_doc(param) + "\n"
            local_name = param['name']
            # some params collide with python keywords, that's why we do
            # this switch (and undo it inside the function we generate)
            if param['name'] == 'except':
                local_name = 'except_'
            original_names[local_name] = param['name']
            keywords.append(local_name)
            if param['required']:
                params_def.append("%s" % local_name)
            else:
                params_def.append("%s=None" % local_name)

        func_head = 'def {0}(self, {1}):'.format(
            as_global and self.get_global_method_name() or self.name,
            ', '.join(params_def)
        )
        code_body = (
            '   _vars_ = locals()\n'
            '   _url = self._fill_url("{url}", _vars_, {url_params})\n'
            '   _original_names = {original_names}\n'
            '   _kwargs = dict((_original_names[k], _vars_[k])\n'
            '                   for k in {keywords} if _vars_[k])\n'
            '   return self._foreman.do_{http_method}(_url, _kwargs)')
        code_body = code_body.format(
            http_method=self.http_method.lower(),
            url=self.url,
            url_params=self.url_params,
            keywords=keywords,
            original_names=original_names,
        )

        code = [
            func_head,
            '   """',
            self.short_desc,
            '',
            params_doc,
            '   """',
            code_body,
        ]

        code = '\n'.join(code)

        six.exec_(code)

        function = locals()[self.name]
        # to ease debugging, all the funcs have the definitions attached
        setattr(function, 'defs', self)
        return function

Response: Generates function for specific method and using specific api


Instruction:
Here you have a function in Python 3, explain what it does
def create_param_doc(cls, param, prefix=None):
        desc = cls.exclude_html_reg.sub('', param['description']).strip()
        if not desc:
            desc = "<no description>"
        name = param['name']
        if prefix:
            name = "%s[%s]" % (prefix, name)
        doc_ = ":param %s: %s; %s" % (name, desc, param['validator'])
        if param['required']:
            doc_ += " (REQUIRED)"
        else:
            doc_ += " (OPTIONAL)"
        for param in param.get('params', []):
            doc_ += "\n" + cls.create_param_doc(param, name)
        return doc_

Response: Generate documentation for single parameter of function


Instruction:
How would you explain what the following Python 3 function does
def convert_plugin_def(http_method, funcs):
        methods = []
        if http_method not in ('GET', 'PUT', 'POST', 'DELETE'):
            logger.error(
                'Plugin load failure, HTTP method %s unsupported.',
                http_method,
            )
            return methods
        for fname, params in six.iteritems(funcs):
            method = {
                'apis': [{'short_description': 'no-doc'}],
                'params': [],
            }
            method['apis'][0]['http_method'] = http_method
            method['apis'][0]['api_url'] = '/api/' + fname
            method['name'] = fname
            for pname, pdef in six.iteritems(params):
                param = {
                    'name': pname,
                    'validator': "Must be %s" % pdef['ptype'],
                    'description': '',
                    'required': pdef['required'],
                }
                method['params'].append(param)
            methods.append(method)
        return methods

Response:
This function parses the plugin definition dict for a
            plugin and extracts the relevant information.


Instruction:
Here you have a function in Python 3, explain what it does
def get_current_version(repo_path):
    repo = dulwich.repo.Repo(repo_path)
    tags = get_tags(repo)
    maj_version = 0
    feat_version = 0
    fix_version = 0

    for commit_sha, children in reversed(
            get_children_per_first_parent(repo_path).items()
    ):
        commit = get_repo_object(repo, commit_sha)
        maj_version, feat_version, fix_version = get_version(
            commit=commit,
            tags=tags,
            maj_version=maj_version,
            feat_version=feat_version,
            fix_version=fix_version,
            children=children,
        )

    return '%s.%s.%s' % (maj_version, feat_version, fix_version)

Response: Given a repo will return the current version of the current version


Instruction:
Given the following Python 3 function, write the documentation
def get_authors(repo_path, from_commit):
    repo = dulwich.repo.Repo(repo_path)
    refs = get_refs(repo)
    start_including = False
    authors = set()

    if from_commit is None:
        start_including = True

    for commit_sha, children in reversed(
        get_children_per_first_parent(repo_path).items()
    ):
        commit = get_repo_object(repo, commit_sha)
        if (
            start_including or commit_sha.startswith(from_commit) or
            fuzzy_matches_refs(from_commit, refs.get(commit_sha, []))
        ):
            authors.add(commit.author.decode())
            for child in children:
                authors.add(child.author.decode())

            start_including = True

    return '\n'.join(sorted(authors))

Response: Given a repo and optionally a base revision to start from will return a list of authors.


Instruction:
Write a Python 3 function that can
emit a spout Tuple message.

Response:
def emit(
        self, tup, tup_id=None, stream=None, direct_task=None, need_task_ids=False
    ):
        """Emit a spout Tuple message.

        :param tup: the Tuple to send to Storm, should contain only
                    JSON-serializable data.
        :type tup: list or tuple
        :param tup_id: the ID for the Tuple. Leave this blank for an
                       unreliable emit.
        :type tup_id: str
        :param stream: ID of the stream this Tuple should be emitted to.
                       Leave empty to emit to the default stream.
        :type stream: str
        :param direct_task: the task to send the Tuple to if performing a
                            direct emit.
        :type direct_task: int
        :param need_task_ids: indicate whether or not you'd like the task IDs
                              the Tuple was emitted (default: ``False``).
        :type need_task_ids: bool

        :returns: ``None``, unless ``need_task_ids=True``, in which case it will
                  be a ``list`` of task IDs that the Tuple was sent to if. Note
                  that when specifying direct_task, this will be equal to
                  ``[direct_task]``.
        """
        return super(Spout, self).emit(
            tup,
            tup_id=tup_id,
            stream=stream,
            direct_task=direct_task,
            need_task_ids=need_task_ids,
        )


Instruction:
Can you implement a function in Python 3 that
calls when a bolt acknowledges a tuple in the topology.

Response:
def ack(self, tup_id):
        """Called when a bolt acknowledges a Tuple in the topology.

        :param tup_id: the ID of the Tuple that has been fully acknowledged in
                       the topology.
        :type tup_id: str
        """
        self.failed_tuples.pop(tup_id, None)
        try:
            del self.unacked_tuples[tup_id]
        except KeyError:
            self.logger.error("Received ack for unknown tuple ID: %r", tup_id)


Instruction:
How would you explain what the following Python 3 function does
def fail(self, tup_id):
        saved_args = self.unacked_tuples.get(tup_id)
        if saved_args is None:
            self.logger.error("Received fail for unknown tuple ID: %r", tup_id)
            return
        tup, stream, direct_task, need_task_ids = saved_args
        if self.failed_tuples[tup_id] < self.max_fails:
            self.emit(
                tup,
                tup_id=tup_id,
                stream=stream,
                direct_task=direct_task,
                need_task_ids=need_task_ids,
            )
            self.failed_tuples[tup_id] += 1
        else:
            # Just pretend we got an ack when we exceed retry limit
            self.logger.info(
                "Acking tuple ID %r after it exceeded retry limit " "(%r)",
                tup_id,
                self.max_fails,
            )
            self.ack(tup_id)

Response: Called when a Tuple fails in the topology.


Instruction:
Here you have a function in Python 3, explain what it does
def emit(
        self, tup, tup_id=None, stream=None, direct_task=None, need_task_ids=False
    ):
        """Emit a spout Tuple & add metadata about it to `unacked_tuples`.

        In order for this to work, `tup_id` is a required parameter.

        See :meth:`Bolt.emit`.
        """
        if tup_id is None:
            raise ValueError(
                "You must provide a tuple ID when emitting with a "
                "ReliableSpout in order for the tuple to be "
                "tracked."
            )
        args = (tup, stream, direct_task, need_task_ids)
        self.unacked_tuples[tup_id] = args
        return super(ReliableSpout, self).emit(
            tup,
            tup_id=tup_id,
            stream=stream,
            direct_task=direct_task,
            need_task_ids=need_task_ids,
        )

Response: Emit a single entry in the unacked_tuples dictionary.


Instruction:
Given the following Python 3 function, write the documentation
def remote_pdb_handler(signum, frame):
    try:
        from remote_pdb import RemotePdb

        rdb = RemotePdb(host="127.0.0.1", port=0)
        rdb.set_trace(frame=frame)
    except ImportError:
        log.warning(
            "remote_pdb unavailable.  Please install remote_pdb to "
            "allow remote debugging."
        )
    # Restore signal handler for later
    signal.signal(signum, remote_pdb_handler)

Response: Handler to drop us into a remote debugger upon receiving SIGUSR1


Instruction:
How would you explain what the following Python 3 function does
def emit(self, record):
        try:
            msg = self.format(record)
            level = _STORM_LOG_LEVELS.get(record.levelname.lower(), _STORM_LOG_INFO)
            self.serializer.send_message(
                {"command": "log", "msg": str(msg), "level": level}
            )
        except Exception:
            self.handleError(record)

Response: Emit a record to Storm.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _setup_component(self, storm_conf, context):
        self.topology_name = storm_conf.get("topology.name", "")
        self.task_id = context.get("taskid", "")
        self.component_name = context.get("componentid")
        # If using Storm before 0.10.0 componentid is not available
        if self.component_name is None:
            self.component_name = context.get("task->component", {}).get(
                str(self.task_id), ""
            )
        self.debug = storm_conf.get("topology.debug", False)
        self.storm_conf = storm_conf
        self.context = context

        # Set up logging
        self.logger = logging.getLogger(".".join((__name__, self.component_name)))
        log_path = self.storm_conf.get("pystorm.log.path")
        log_file_name = self.storm_conf.get(
            "pystorm.log.file",
            "pystorm_{topology_name}" "_{component_name}" "_{task_id}" "_{pid}.log",
        )
        root_log = logging.getLogger()
        log_level = self.storm_conf.get("pystorm.log.level", "info")
        if log_path:
            max_bytes = self.storm_conf.get("pystorm.log.max_bytes", 1000000)  # 1 MB
            backup_count = self.storm_conf.get("pystorm.log.backup_count", 10)
            log_file = join(
                log_path,
                (
                    log_file_name.format(
                        topology_name=self.topology_name,
                        component_name=self.component_name,
                        task_id=self.task_id,
                        pid=self.pid,
                    )
                ),
            )
            handler = RotatingFileHandler(
                log_file, maxBytes=max_bytes, backupCount=backup_count
            )
            log_format = self.storm_conf.get(
                "pystorm.log.format",
                "%(asctime)s - %(name)s - " "%(levelname)s - %(message)s",
            )
        else:
            self.log(
                "pystorm StormHandler logging enabled, so all messages at "
                'levels greater than "pystorm.log.level" ({}) will be sent'
                " to Storm.".format(log_level)
            )
            handler = StormHandler(self.serializer)
            log_format = self.storm_conf.get(
                "pystorm.log.format", "%(asctime)s - %(name)s - " "%(message)s"
            )
        formatter = logging.Formatter(log_format)
        log_level = _PYTHON_LOG_LEVELS.get(log_level, logging.INFO)
        if self.debug:
            # potentially override logging that was provided if
            # topology.debug was set to true
            log_level = logging.DEBUG
        handler.setLevel(log_level)
        handler.setFormatter(formatter)
        root_log.addHandler(handler)
        self.logger.setLevel(log_level)
        logging.getLogger("pystorm").setLevel(log_level)
        # Redirect stdout to ensure that print statements/functions
        # won't disrupt the multilang protocol
        if self.serializer.output_stream == sys.stdout:
            sys.stdout = LogStream(logging.getLogger("pystorm.stdout"))

Response: Setup the component with the given storm_conf and context.


Instruction:
How would you code a function in Python 3 to
read and process an initial handshake message from Storm.

Response:
def read_handshake(self):
        """Read and process an initial handshake message from Storm."""
        msg = self.read_message()
        pid_dir, _conf, _context = msg["pidDir"], msg["conf"], msg["context"]

        # Write a blank PID file out to the pidDir
        open(join(pid_dir, str(self.pid)), "w").close()
        self.send_message({"pid": self.pid})

        return _conf, _context


Instruction:
Create a Python 3 function to
send a message to Storm via stdout.

Response:
def send_message(self, message):
        """Send a message to Storm via stdout."""
        if not isinstance(message, dict):
            logger = self.logger if self.logger else log
            logger.error(
                "%s.%d attempted to send a non dict message to Storm: " "%r",
                self.component_name,
                self.pid,
                message,
            )
            return
        self.serializer.send_message(message)


Instruction:
Create a Python 3 function to
report an exception back to Storm via logging.

Response:
def raise_exception(self, exception, tup=None):
        """Report an exception back to Storm via logging.

        :param exception: a Python exception.
        :param tup: a :class:`Tuple` object.
        """
        if tup:
            message = (
                "Python {exception_name} raised while processing Tuple "
                "{tup!r}\n{traceback}"
            )
        else:
            message = "Python {exception_name} raised\n{traceback}"
        message = message.format(
            exception_name=exception.__class__.__name__, tup=tup, traceback=format_exc()
        )
        self.send_message({"command": "error", "msg": str(message)})
        self.send_message({"command": "sync"})


Instruction:
Write a Python 3 function for
logging a message to Storm optionally providing a logging level.

Response:
def log(self, message, level=None):
        """Log a message to Storm optionally providing a logging level.

        :param message: the log message to send to Storm.
        :type message: str
        :param level: the logging level that Storm should use when writing the
                      ``message``. Can be one of: trace, debug, info, warn, or
                      error (default: ``info``).
        :type level: str

        .. warning::

          This will send your message to Storm regardless of what level you
          specify.  In almost all cases, you are better of using
          ``Component.logger`` and not setting ``pystorm.log.path``, because
          that will use a :class:`pystorm.component.StormHandler` to do the
          filtering on the Python side (instead of on the Java side after taking
          the time to serialize your message and send it to Storm).
        """
        level = _STORM_LOG_LEVELS.get(level, _STORM_LOG_INFO)
        self.send_message({"command": "log", "msg": str(message), "level": level})


Instruction:
Write a Python 3 function for
emitting a new tuple to a stream.

Response:
def emit(
        self,
        tup,
        tup_id=None,
        stream=None,
        anchors=None,
        direct_task=None,
        need_task_ids=False,
    ):
        """Emit a new Tuple to a stream.

        :param tup: the Tuple payload to send to Storm, should contain only
                    JSON-serializable data.
        :type tup: :class:`list` or :class:`pystorm.component.Tuple`
        :param tup_id: the ID for the Tuple. If omitted by a
                       :class:`pystorm.spout.Spout`, this emit will be
                       unreliable.
        :type tup_id: str
        :param stream: the ID of the stream to emit this Tuple to. Specify
                       ``None`` to emit to default stream.
        :type stream: str
        :param anchors: IDs the Tuples (or
                        :class:`pystorm.component.Tuple` instances)
                        which the emitted Tuples should be anchored to. This is
                        only passed by :class:`pystorm.bolt.Bolt`.
        :type anchors: list
        :param direct_task: the task to send the Tuple to.
        :type direct_task: int
        :param need_task_ids: indicate whether or not you'd like the task IDs
                              the Tuple was emitted (default: ``False``).
        :type need_task_ids: bool

        :returns: ``None``, unless ``need_task_ids=True``, in which case it will
                  be a ``list`` of task IDs that the Tuple was sent to if. Note
                  that when specifying direct_task, this will be equal to
                  ``[direct_task]``.
        """
        if not isinstance(tup, (list, tuple)):
            raise TypeError(
                "All Tuples must be either lists or tuples, "
                "received {!r} instead.".format(type(tup))
            )

        msg = {"command": "emit", "tuple": tup}
        downstream_task_ids = None

        if anchors is not None:
            msg["anchors"] = anchors
        if tup_id is not None:
            msg["id"] = tup_id
        if stream is not None:
            msg["stream"] = stream
        if direct_task is not None:
            msg["task"] = direct_task
            if need_task_ids:
                downstream_task_ids = [direct_task]

        if not need_task_ids:
            # only need to send on False, Storm's default is True
            msg["need_task_ids"] = need_task_ids

        if need_task_ids and direct_task is None:
            # Use both locks so we ensure send_message and read_task_ids are for
            # same emit
            with self._reader_lock, self._writer_lock:
                self.send_message(msg)
                downstream_task_ids = self.read_task_ids()
        # No locks necessary in simple case because serializer will acquire
        # write lock itself
        else:
            self.send_message(msg)

        return downstream_task_ids


Instruction:
Can you generate the documentation for the following Python 3 function
def _wrap_stream(stream):
        if hasattr(stream, "buffer"):
            return io.TextIOWrapper(stream.buffer, encoding="utf-8")
        elif hasattr(stream, "readable"):
            return io.TextIOWrapper(stream, encoding="utf-8")
        # Python 2.x stdin and stdout are just files
        else:
            return io.open(stream.fileno(), mode=stream.mode, encoding="utf-8")

Response:
Returns a TextIOWrapper around the given stream that handles UTF - 8
        encoding and decoding.


Instruction:
Can you write a function in Python 3 where it
reads a message from the input stream and returns it as a dict.

Response:
def read_message(self):
        """The Storm multilang protocol consists of JSON messages followed by
        a newline and "end\n".

        All of Storm's messages (for either bolts or spouts) should be of the
        form::

            '<command or task_id form prior emit>\\nend\\n'

        Command example, an incoming Tuple to a bolt::

            '{ "id": "-6955786537413359385",  "comp": "1", "stream": "1", "task": 9, "tuple": ["snow white and the seven dwarfs", "field2", 3]}\\nend\\n'

        Command example for a spout to emit its next Tuple::

            '{"command": "next"}\\nend\\n'

        Example, the task IDs a prior emit was sent to::

            '[12, 22, 24]\\nend\\n'

        The edge case of where we read ``''`` from ``input_stream`` indicating
        EOF, usually means that communication with the supervisor has been
        severed.
        """
        msg = ""
        num_blank_lines = 0
        while True:
            # readline will return trailing \n so that output is unambigious, we
            # should only have line == '' if we're at EOF
            with self._reader_lock:
                line = self.input_stream.readline()
            if line == "end\n":
                break
            elif line == "":
                raise StormWentAwayError()
            elif line == "\n":
                num_blank_lines += 1
                if num_blank_lines % 1000 == 0:
                    log.warn(
                        "While trying to read a command or pending task "
                        "ID, Storm has instead sent %s '\\n' messages.",
                        num_blank_lines,
                    )
                continue

            msg = "{}{}\n".format(msg, line[0:-1])

        try:
            return json.loads(msg)
        except Exception:
            log.error("JSON decode error for message: %r", msg, exc_info=True)
            raise


Instruction:
Write a Python 3 function for
serializing to JSON a message dictionary.

Response:
def serialize_dict(self, msg_dict):
        """Serialize to JSON a message dictionary."""
        serialized = json.dumps(msg_dict, namedtuple_as_object=False)
        if PY2:
            serialized = serialized.decode("utf-8")
        serialized = "{}\nend\n".format(serialized)
        return serialized


Instruction:
Implement a Python 3 function for
reading a tuple from the pipe to Storm.

Response:
def read_tuple(self):
        """Read a tuple from the pipe to Storm."""
        cmd = self.read_command()
        source = cmd["comp"]
        stream = cmd["stream"]
        values = cmd["tuple"]
        val_type = self._source_tuple_types[source].get(stream)
        return Tuple(
            cmd["id"],
            source,
            stream,
            cmd["task"],
            tuple(values) if val_type is None else val_type(*values),
        )


Instruction:
How would you code a function in Python 3 to
emit a new tuple to Storm.

Response:
def emit(
        self, tup, stream=None, anchors=None, direct_task=None, need_task_ids=False
    ):
        """Emit a new Tuple to a stream.

        :param tup: the Tuple payload to send to Storm, should contain only
                    JSON-serializable data.
        :type tup: :class:`list` or :class:`pystorm.component.Tuple`
        :param stream: the ID of the stream to emit this Tuple to. Specify
                       ``None`` to emit to default stream.
        :type stream: str
        :param anchors: IDs the Tuples (or :class:`pystorm.component.Tuple`
                        instances) which the emitted Tuples should be anchored
                        to. If ``auto_anchor`` is set to ``True`` and
                        you have not specified ``anchors``, ``anchors`` will be
                        set to the incoming/most recent Tuple ID(s).
        :type anchors: list
        :param direct_task: the task to send the Tuple to.
        :type direct_task: int
        :param need_task_ids: indicate whether or not you'd like the task IDs
                              the Tuple was emitted (default: ``False``).
        :type need_task_ids: bool

        :returns: ``None``, unless ``need_task_ids=True``, in which case it will
                  be a ``list`` of task IDs that the Tuple was sent to if. Note
                  that when specifying direct_task, this will be equal to
                  ``[direct_task]``.
        """
        if anchors is None:
            anchors = self._current_tups if self.auto_anchor else []
        anchors = [a.id if isinstance(a, Tuple) else a for a in anchors]

        return super(Bolt, self).emit(
            tup,
            stream=stream,
            anchors=anchors,
            direct_task=direct_task,
            need_task_ids=need_task_ids,
        )


Instruction:
Can you create a Python 3 function that
indicates that processing of a Tuple has succeeded.

Response:
def ack(self, tup):
        """Indicate that processing of a Tuple has succeeded.

        :param tup: the Tuple to acknowledge.
        :type tup: :class:`str` or :class:`pystorm.component.Tuple`
        """
        tup_id = tup.id if isinstance(tup, Tuple) else tup
        self.send_message({"command": "ack", "id": tup_id})


Instruction:
Write a Python 3 function for
indicating that processing of a Tuple has failed.

Response:
def fail(self, tup):
        """Indicate that processing of a Tuple has failed.

        :param tup: the Tuple to fail (its ``id`` if ``str``).
        :type tup: :class:`str` or :class:`pystorm.component.Tuple`
        """
        tup_id = tup.id if isinstance(tup, Tuple) else tup
        self.send_message({"command": "fail", "id": tup_id})


Instruction:
Implement a Python 3 function for
handling an exception raised while running the run loop.

Response:
def _handle_run_exception(self, exc):
        """Process an exception encountered while running the ``run()`` loop.

        Called right before program exits.
        """
        if len(self._current_tups) == 1:
            tup = self._current_tups[0]
            self.raise_exception(exc, tup)
            if self.auto_fail:
                self.fail(tup)


Instruction:
Can you write a function in Python 3 where it
modifies emit that will not return task IDs after emitting.

Response:
def emit(self, tup, **kwargs):
        """Modified emit that will not return task IDs after emitting.

        See :class:`pystorm.component.Bolt` for more information.

        :returns: ``None``.
        """
        kwargs["need_task_ids"] = False
        return super(BatchingBolt, self).emit(tup, **kwargs)


Instruction:
Can you generate the documentation for the following Python 3 function
def process_tick(self, tick_tup):
        self._tick_counter += 1
        # ACK tick Tuple immediately, since it's just responsible for counter
        self.ack(tick_tup)
        if self._tick_counter > self.ticks_between_batches and self._batches:
            self.process_batches()
            self._tick_counter = 0

Response: Increment tick counter and call process_batch for all current Tuples.


Instruction:
Can you implement a function in Python 3 that
iterates through all batches and process them and ack.

Response:
def process_batches(self):
        """Iterate through all batches, call process_batch on them, and ack.

        Separated out for the rare instances when we want to subclass
        BatchingBolt and customize what mechanism causes batches to be
        processed.
        """
        for key, batch in iteritems(self._batches):
            self._current_tups = batch
            self._current_key = key
            self.process_batch(key, batch)
            if self.auto_ack:
                for tup in batch:
                    self.ack(tup)
            # Set current batch to [] so that we know it was acked if a
            # later batch raises an exception
            self._current_key = None
            self._batches[key] = []
        self._batches = defaultdict(list)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def process(self, tup):
        # Append latest Tuple to batches
        group_key = self.group_key(tup)
        self._batches[group_key].append(tup)

Response: Group non - tick Tuples into batches by group_key.


Instruction:
Can you generate the documentation for the following Python 3 function
def _handle_run_exception(self, exc):
        self.raise_exception(exc, self._current_tups)

        if self.auto_fail:
            failed = set()
            for key, batch in iteritems(self._batches):
                # Only wipe out batches other than current for exit_on_exception
                if self.exit_on_exception or key == self._current_key:
                    for tup in batch:
                        self.fail(tup)
                        failed.add(tup.id)

            # Fail current batch or tick Tuple if we have one
            for tup in self._current_tups:
                if tup.id not in failed:
                    self.fail(tup)

            # Reset current batch info
            self._batches[self._current_key] = []
            self._current_key = None

Response: Handle an exception raised while running the run loop.


Instruction:
Make a summary of the following Python 3 code
def _batch_entry_run(self):
        time.sleep(self.secs_between_batches)
        with self._batch_lock:
            self.process_batches()

Response: The inside of _batch_entry s infinite loop.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _batch_entry(self):
        try:
            while True:
                self._batch_entry_run()
        except:
            self.exc_info = sys.exc_info()
            os.kill(self.pid, signal.SIGUSR1)

Response: Entry point for the batcher thread.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _run(self):
        tup = self.read_tuple()
        with self._batch_lock:
            self._current_tups = [tup]
            if self.is_heartbeat(tup):
                self.send_message({"command": "sync"})
            elif self.is_tick(tup):
                self.process_tick(tup)
            else:
                self.process(tup)
            # reset so that we don't accidentally fail the wrong Tuples
            # if a successive call to read_tuple fails
            self._current_tups = []

Response:
This method is called by the BatchingBolt s main thread to process the current Tuples
           .


Instruction:
Implement a Python 3 function for
serializing a message dictionary and write it to the output stream.

Response:
def send_message(self, msg_dict):
        """Serialize a message dictionary and write it to the output stream."""
        with self._writer_lock:
            try:
                self.output_stream.flush()
                self.output_stream.write(self.serialize_dict(msg_dict))
                self.output_stream.flush()
            except IOError:
                raise StormWentAwayError()
            except:
                log.exception("Failed to send message: %r", msg_dict)


Instruction:
Write a Python 3 script for
converting the FFI result to Python data structures.

Response:
def _void_array_to_list(restuple, _func, _args):
    """ Convert the FFI result to Python data structures """
    shape = (restuple.e.len, 1)
    array_size = np.prod(shape)
    mem_size = 8 * array_size

    array_str_e = string_at(restuple.e.data, mem_size)
    array_str_n = string_at(restuple.n.data, mem_size)

    ls_e = np.frombuffer(array_str_e, float, array_size).tolist()
    ls_n = np.frombuffer(array_str_n, float, array_size).tolist()

    return ls_e, ls_n


Instruction:
Can you create a Python 3 function that
loads a data file and return it as a list of lines.

Response:
def load_data_file(filename, encoding='utf-8'):
    """Load a data file and return it as a list of lines.

    Parameters:
        filename: The name of the file (no directories included).
        encoding: The file encoding. Defaults to utf-8.

    """
    data = pkgutil.get_data(PACKAGE_NAME, os.path.join(DATA_DIR, filename))
    return data.decode(encoding).splitlines()


Instruction:
Can you create a Python 3 function that
loads the word and character mapping data into a dictionary.

Response:
def _load_data():
    """Load the word and character mapping data into a dictionary.

    In the data files, each line is formatted like this:
        HANZI   PINYIN_READING/PINYIN_READING

    So, lines need to be split by '\t' and then the Pinyin readings need to be
    split by '/'.

    """
    data = {}
    for name, file_name in (('words', 'hanzi_pinyin_words.tsv'),
                            ('characters', 'hanzi_pinyin_characters.tsv')):
        # Split the lines by tabs: [[hanzi, pinyin]...].
        lines = [line.split('\t') for line in
                 dragonmapper.data.load_data_file(file_name)]
        # Make a dictionary: {hanzi: [pinyin, pinyin]...}.
        data[name] = {hanzi: pinyin.split('/') for hanzi, pinyin in lines}
    return data


Instruction:
Can you implement a function in Python 3 that
returns the Pinyin reading for a Chinese word.

Response:
def _hanzi_to_pinyin(hanzi):
    """Return the Pinyin reading for a Chinese word.

    If the given string *hanzi* matches a CC-CEDICT word, the return value is
    formatted like this: [WORD_READING1, WORD_READING2, ...]

    If the given string *hanzi* doesn't match a CC-CEDICT word, the return
    value is formatted like this: [[CHAR_READING1, CHAR_READING2 ...], ...]

    When returning character readings, if a character wasn't recognized, the
    original character is returned, e.g. [[CHAR_READING1, ...], CHAR, ...]

    """
    try:
        return _HANZI_PINYIN_MAP['words'][hanzi]
    except KeyError:
        return [_CHARACTERS.get(character, character) for character in hanzi]


Instruction:
Can you implement a function in Python 3 that
encloses a reading within a container.

Response:
def _enclose_readings(container, readings):
    """Enclose a reading within a container, e.g. '[]'."""
    container_start, container_end = tuple(container)
    enclosed_readings = '%(container_start)s%(readings)s%(container_end)s' % {
        'container_start': container_start, 'container_end': container_end,
        'readings': readings}
    return enclosed_readings


Instruction:
Explain what the following Python 3 code does
def to_pinyin(s, delimiter=' ', all_readings=False, container='[]',
              accented=True):
    """Convert a string's Chinese characters to Pinyin readings.

    *s* is a string containing Chinese characters. *accented* is a
    boolean value indicating whether to return accented or numbered Pinyin
    readings.

    *delimiter* is the character used to indicate word boundaries in *s*.
    This is used to differentiate between words and characters so that a more
    accurate reading can be returned.

    *all_readings* is a boolean value indicating whether or not to return all
    possible readings in the case of words/characters that have multiple
    readings. *container* is a two character string that is used to
    enclose words/characters if *all_readings* is ``True``. The default
    ``'[]'`` is used like this: ``'[READING1/READING2]'``.

    Characters not recognized as Chinese are left untouched.

    """
    hanzi = s
    pinyin = ''

    # Process the given string.
    while hanzi:

        # Get the next match in the given string.
        match = re.search('[^%s%s]+' % (delimiter, zhon.hanzi.punctuation),
                          hanzi)

        # There are no more matches, but the string isn't finished yet.
        if match is None and hanzi:
            pinyin += hanzi
            break

        match_start, match_end = match.span()

        # Process the punctuation marks that occur before the match.
        if match_start > 0:
            pinyin += hanzi[0:match_start]

        # Get the Chinese word/character readings.
        readings = _hanzi_to_pinyin(match.group())

        # Process the returned word readings.
        if match.group() in _WORDS:
            if all_readings:
                reading = _enclose_readings(container,
                                            _READING_SEPARATOR.join(readings))
            else:
                reading = readings[0]
            pinyin += reading

        # Process the returned character readings.
        else:
            # Process each character individually.
            for character in readings:
                # Don't touch unrecognized characters.
                if isinstance(character, str):
                    pinyin += character
                # Format multiple readings.
                elif isinstance(character, list) and all_readings:
                    pinyin += _enclose_readings(
                        container, _READING_SEPARATOR.join(character))
                # Select and format the most common reading.
                elif isinstance(character, list) and not all_readings:
                    # Add an apostrophe to separate syllables.
                    if (pinyin and character[0][0] in zhon.pinyin.vowels and
                            pinyin[-1] in zhon.pinyin.lowercase):
                        pinyin += "'"
                    pinyin += character[0]

        # Move ahead in the given string.
        hanzi = hanzi[match_end:]

    if accented:
        return pinyin
    else:
        return accented_to_numbered(pinyin)

Response: Convert a string s Chinese characters to Pinyin readings.


Instruction:
Write a Python 3 function that can
convert a string s Chinese characters to Zhuyin readings.

Response:
def to_zhuyin(s, delimiter=' ', all_readings=False, container='[]'):
    """Convert a string's Chinese characters to Zhuyin readings.

    *s* is a string containing Chinese characters.

    *delimiter* is the character used to indicate word boundaries in *s*.
    This is used to differentiate between words and characters so that a more
    accurate reading can be returned.

    *all_readings* is a boolean value indicating whether or not to return all
    possible readings in the case of words/characters that have multiple
    readings. *container* is a two character string that is used to
    enclose words/characters if *all_readings* is ``True``. The default
    ``'[]'`` is used like this: ``'[READING1/READING2]'``.

    Characters not recognized as Chinese are left untouched.

    """
    numbered_pinyin = to_pinyin(s, delimiter, all_readings, container, False)
    zhuyin = pinyin_to_zhuyin(numbered_pinyin)
    return zhuyin


Instruction:
Can you tell what is the following Python 3 function doing
def to_ipa(s, delimiter=' ', all_readings=False, container='[]'):
    numbered_pinyin = to_pinyin(s, delimiter, all_readings, container, False)
    ipa = pinyin_to_ipa(numbered_pinyin)
    return ipa

Response: Convert a string s Chinese characters to IPA.


Instruction:
Create a Python 3 function for
loading the transcription mapping data into a dictionary.

Response:
def _load_data():
    """Load the transcription mapping data into a dictionary."""
    lines = dragonmapper.data.load_data_file('transcriptions.csv')
    pinyin_map, zhuyin_map, ipa_map = {}, {}, {}
    for line in lines:
        p, z, i = line.split(',')
        pinyin_map[p] = {'Zhuyin': z, 'IPA': i}
        zhuyin_map[z] = {'Pinyin': p, 'IPA': i}
        ipa_map[i] = {'Pinyin': p, 'Zhuyin': z}
    return pinyin_map, zhuyin_map, ipa_map


Instruction:
Can you implement a function in Python 3 that
converts a numbered Pinyin vowel to an accented Pinyin vowel.

Response:
def _numbered_vowel_to_accented(vowel, tone):
    """Convert a numbered Pinyin vowel to an accented Pinyin vowel."""
    if isinstance(tone, int):
        tone = str(tone)
    return _PINYIN_TONES[vowel + tone]


Instruction:
Can you write a function in Python 3 where it
converts an accented Pinyin vowel to a numbered Pinyin vowel.

Response:
def _accented_vowel_to_numbered(vowel):
    """Convert an accented Pinyin vowel to a numbered Pinyin vowel."""
    for numbered_vowel, accented_vowel in _PINYIN_TONES.items():
        if vowel == accented_vowel:
            return tuple(numbered_vowel)


Instruction:
Implement a Python 3 function for
returning the syllable and tone of a numbered Pinyin syllable.

Response:
def _parse_numbered_syllable(unparsed_syllable):
    """Return the syllable and tone of a numbered Pinyin syllable."""
    tone_number = unparsed_syllable[-1]
    if not tone_number.isdigit():
        syllable, tone = unparsed_syllable, '5'
    elif tone_number == '0':
        syllable, tone = unparsed_syllable[:-1], '5'
    elif tone_number in '12345':
        syllable, tone = unparsed_syllable[:-1], tone_number
    else:
        raise ValueError("Invalid syllable: %s" % unparsed_syllable)
    return syllable, tone


Instruction:
Here you have a function in Python 3, explain what it does
def _parse_accented_syllable(unparsed_syllable):
    if unparsed_syllable[0] == '\u00B7':
        # Special case for middle dot tone mark.
        return unparsed_syllable[1:], '5'
    for character in unparsed_syllable:
        if character in _ACCENTED_VOWELS:
            vowel, tone = _accented_vowel_to_numbered(character)
            return unparsed_syllable.replace(character, vowel), tone
    return unparsed_syllable, '5'

Response: Return the syllable and tone of an accented Pinyin syllable.


Instruction:
Create a Python 3 function for
returning the syllable and tone of a Zhuyin syllable.

Response:
def _parse_zhuyin_syllable(unparsed_syllable):
    """Return the syllable and tone of a Zhuyin syllable."""
    zhuyin_tone = unparsed_syllable[-1]
    if zhuyin_tone in zhon.zhuyin.characters:
        syllable, tone = unparsed_syllable, '1'
    elif zhuyin_tone in zhon.zhuyin.marks:
        for tone_number, tone_mark in _ZHUYIN_TONES.items():
            if zhuyin_tone == tone_mark:
                syllable, tone = unparsed_syllable[:-1], tone_number
    else:
        raise ValueError("Invalid syllable: %s" % unparsed_syllable)

    return syllable, tone


Instruction:
Explain what the following Python 3 code does
def _parse_ipa_syllable(unparsed_syllable):
    ipa_tone = re.search('[%(marks)s]+' % {'marks': _IPA_MARKS},
                         unparsed_syllable)
    if not ipa_tone:
        syllable, tone = unparsed_syllable, '5'
    else:
        for tone_number, tone_mark in _IPA_TONES.items():
            if ipa_tone.group() == tone_mark:
                tone = tone_number
                break
        syllable = unparsed_syllable[0:ipa_tone.start()]
    return syllable, tone

Response: Return the syllable and tone of an IPA syllable.


Instruction:
How would you code a function in Python 3 to
restore a lowercase string s characters to their original case.

Response:
def _restore_case(s, memory):
    """Restore a lowercase string's characters to their original case."""
    cased_s = []
    for i, c in enumerate(s):
        if i + 1 > len(memory):
            break
        cased_s.append(c if memory[i] else c.upper())
    return ''.join(cased_s)


Instruction:
Implement a function in Python 3 to
convert a numbered Pinyin syllable to an accented Pinyin syllable.

Response:
def numbered_syllable_to_accented(s):
    """Convert numbered Pinyin syllable *s* to an accented Pinyin syllable.

    It implements the following algorithm to determine where to place tone
    marks:

    1. If the syllable has an 'a', 'e', or 'o' (in that order), put the
        tone mark over that vowel.
    2. Otherwise, put the tone mark on the last vowel.

    """
    if s == 'r5':
        return 'r'  # Special case for 'r' suffix.

    lowercase_syllable, case_memory = _lower_case(s)
    syllable, tone = _parse_numbered_syllable(lowercase_syllable)
    syllable = syllable.replace('v', '\u00fc')
    if re.search('[%s]' % _UNACCENTED_VOWELS, syllable) is None:
        return s
    if 'a' in syllable:
        accented_a = _numbered_vowel_to_accented('a', tone)
        accented_syllable = syllable.replace('a', accented_a)
    elif 'e' in syllable:
        accented_e = _numbered_vowel_to_accented('e', tone)
        accented_syllable = syllable.replace('e', accented_e)
    elif 'o' in syllable:
        accented_o = _numbered_vowel_to_accented('o', tone)
        accented_syllable = syllable.replace('o', accented_o)
    else:
        vowel = syllable[max(map(syllable.rfind, _UNACCENTED_VOWELS))]
        accented_vowel = _numbered_vowel_to_accented(vowel, tone)
        accented_syllable = syllable.replace(vowel, accented_vowel)
    return _restore_case(accented_syllable, case_memory)


Instruction:
Write a Python 3 script to
convert accented Pinyin syllable s to a numbered Pinyin syllable.

Response:
def accented_syllable_to_numbered(s):
    """Convert accented Pinyin syllable *s* to a numbered Pinyin syllable."""
    if s[0] == '\u00B7':
        lowercase_syllable, case_memory = _lower_case(s[1:])
        lowercase_syllable = '\u00B7' + lowercase_syllable
    else:
        lowercase_syllable, case_memory = _lower_case(s)
    numbered_syllable, tone = _parse_accented_syllable(lowercase_syllable)
    return _restore_case(numbered_syllable, case_memory) + tone


Instruction:
How would you code a function in Python 3 to
convert a Pinyin syllable to a Zhuyin syllable.

Response:
def pinyin_syllable_to_zhuyin(s):
    """Convert Pinyin syllable *s* to a Zhuyin syllable."""
    pinyin_syllable, tone = _parse_pinyin_syllable(s)
    try:
        zhuyin_syllable = _PINYIN_MAP[pinyin_syllable.lower()]['Zhuyin']
    except KeyError:
        raise ValueError('Not a valid syllable: %s' % s)
    return zhuyin_syllable + _ZHUYIN_TONES[tone]


Instruction:
Given the following Python 3 function, write the documentation
def pinyin_syllable_to_ipa(s):
    pinyin_syllable, tone = _parse_pinyin_syllable(s)
    try:
        ipa_syllable = _PINYIN_MAP[pinyin_syllable.lower()]['IPA']
    except KeyError:
        raise ValueError('Not a valid syllable: %s' % s)
    return ipa_syllable + _IPA_TONES[tone]

Response: Convert Pinyin syllable to an IPA syllable.


Instruction:
Write a Python 3 script for
converting a Zhuyin syllable to a numbered Pinyin syllable.

Response:
def _zhuyin_syllable_to_numbered(s):
    """Convert Zhuyin syllable *s* to a numbered Pinyin syllable."""
    zhuyin_syllable, tone = _parse_zhuyin_syllable(s)
    try:
        pinyin_syllable = _ZHUYIN_MAP[zhuyin_syllable]['Pinyin']
    except KeyError:
        raise ValueError('Not a valid syllable: %s' % s)
    return pinyin_syllable + tone


Instruction:
How would you code a function in Python 3 to
convert an IPA syllable to a numbered Pinyin syllable.

Response:
def _ipa_syllable_to_numbered(s):
    """Convert IPA syllable *s* to a numbered Pinyin syllable."""
    ipa_syllable, tone = _parse_ipa_syllable(s)
    try:
        pinyin_syllable = _IPA_MAP[ipa_syllable]['Pinyin']
    except KeyError:
        raise ValueError('Not a valid syllable: %s' % s)
    return pinyin_syllable + tone


Instruction:
How would you implement a function in Python 3 that
converts a string s syllables to a different transcription system.

Response:
def _convert(s, re_pattern, syllable_function, add_apostrophes=False,
             remove_apostrophes=False, separate_syllables=False):
    """Convert a string's syllables to a different transcription system."""
    original = s
    new = ''
    while original:
        match = re.search(re_pattern, original, re.IGNORECASE | re.UNICODE)
        if match is None and original:
            # There are no more matches, but the given string isn't fully
            # processed yet.
            new += original
            break
        match_start, match_end = match.span()
        if match_start > 0:  # Handle extra characters before matched syllable.
            if (new and remove_apostrophes and match_start == 1 and
                    original[0] == "'"):
                pass  # Remove the apostrophe between Pinyin syllables.
                if separate_syllables:  # Separate syllables by a space.
                    new += ' '
            else:
                new += original[0:match_start]
        else:  # Matched syllable starts immediately.
            if new and separate_syllables:  # Separate syllables by a space.
                new += ' '
            elif (new and add_apostrophes and
                    match.group()[0].lower() in _UNACCENTED_VOWELS):
                new += "'"
        # Convert the matched syllable.
        new += syllable_function(match.group())
        original = original[match_end:]
    return new


Instruction:
Can you generate a brief explanation for the following Python 3 code
def numbered_to_accented(s):
    return _convert(s, zhon.pinyin.syllable, numbered_syllable_to_accented,
                    add_apostrophes=True)

Response: Convert all numbered Pinyin syllables in * s * to accented Pinyin.


Instruction:
Can you write a function in Python 3 where it
converts all Pinyin syllables in s to Zhuyin.

Response:
def pinyin_to_zhuyin(s):
    """Convert all Pinyin syllables in *s* to Zhuyin.

    Spaces are added between connected syllables and syllable-separating
    apostrophes are removed.

    """
    return _convert(s, zhon.pinyin.syllable, pinyin_syllable_to_zhuyin,
                    remove_apostrophes=True, separate_syllables=True)


Instruction:
How would you implement a function in Python 3 that
converts all Pinyin syllables in s to IPA.

Response:
def pinyin_to_ipa(s):
    """Convert all Pinyin syllables in *s* to IPA.

    Spaces are added between connected syllables and syllable-separating
    apostrophes are removed.

    """
    return _convert(s, zhon.pinyin.syllable, pinyin_syllable_to_ipa,
                    remove_apostrophes=True, separate_syllables=True)


Instruction:
Make a summary of the following Python 3 code
def zhuyin_to_pinyin(s, accented=True):
    if accented:
        function = _zhuyin_syllable_to_accented
    else:
        function = _zhuyin_syllable_to_numbered
    return _convert(s, zhon.zhuyin.syllable, function)

Response: Convert all Zhuyin syllables in s to Pinyin syllables.


Instruction:
How would you implement a function in Python 3 that
converts all IPA syllables in s to Pinyin syllables.

Response:
def ipa_to_pinyin(s, accented=True):
    """Convert all IPA syllables in *s* to Pinyin.

    If *accented* is ``True``, diacritics are added to the Pinyin syllables. If
    it's ``False``, numbers are used to indicate tone.

    """
    if accented:
        function = _ipa_syllable_to_accented
    else:
        function = _ipa_syllable_to_numbered
    return _convert(s, _IPA_SYLLABLE, function)


Instruction:
Implement a function in Python 3 to
convert a Chinese transcription string to a Pinyin syllab.

Response:
def to_pinyin(s, accented=True):
    """Convert *s* to Pinyin.

    If *accented* is ``True``, diacritics are added to the Pinyin syllables. If
    it's ``False``, numbers are used to indicate tone.

    """
    identity = identify(s)
    if identity == PINYIN:
        if _has_accented_vowels(s):
            return s if accented else accented_to_numbered(s)
        else:
            return numbered_to_accented(s) if accented else s
    elif identity == ZHUYIN:
        return zhuyin_to_pinyin(s, accented=accented)
    elif identity == IPA:
        return ipa_to_pinyin(s, accented=accented)
    else:
        raise ValueError("String is not a valid Chinese transcription.")


Instruction:
Implement a function in Python 3 to
convert string s to Zhuyin.

Response:
def to_zhuyin(s):
    """Convert *s* to Zhuyin."""
    identity = identify(s)
    if identity == ZHUYIN:
        return s
    elif identity == PINYIN:
        return pinyin_to_zhuyin(s)
    elif identity == IPA:
        return ipa_to_zhuyin(s)
    else:
        raise ValueError("String is not a valid Chinese transcription.")


Instruction:
Make a summary of the following Python 3 code
def to_ipa(s):
    identity = identify(s)
    if identity == IPA:
        return s
    elif identity == PINYIN:
        return pinyin_to_ipa(s)
    elif identity == ZHUYIN:
        return zhuyin_to_ipa(s)
    else:
        raise ValueError("String is not a valid Chinese transcription.")

Response: Convert string s to IPA.


Instruction:
Here you have a function in Python 3, explain what it does
def _is_pattern_match(re_pattern, s):
    match = re.match(re_pattern, s, re.I)
    return match.group() == s if match else False

Response: Check if a re pattern expression matches an entire string.


Instruction:
Write a Python 3 script to
check if *s* consists of valid Pinyin.

Response:
def is_pinyin(s):
    """Check if *s* consists of valid Pinyin."""
    re_pattern = ('(?:%(word)s|[ \t%(punctuation)s])+' %
                  {'word': zhon.pinyin.word,
                   'punctuation': zhon.pinyin.punctuation})
    return _is_pattern_match(re_pattern, s)


Instruction:
Create a Python 3 function for
checking if a string s contains Zhuyin - compatible characters.

Response:
def is_zhuyin_compatible(s):
    """Checks if *s* is consists of Zhuyin-compatible characters.

    This does not check if *s* contains valid Zhuyin syllables; for that
    see :func:`is_zhuyin`.

    Besides Zhuyin characters and tone marks, spaces are also accepted.
    This function checks that all characters in *s* exist in
    :data:`zhon.zhuyin.characters`, :data:`zhon.zhuyin.marks`, or ``' '``.

    """
    printable_zhuyin = zhon.zhuyin.characters + zhon.zhuyin.marks + ' '
    return _is_pattern_match('[%s]+' % printable_zhuyin, s)


Instruction:
Write a Python 3 function that can
check if *s* consists of valid Chinese IPA.

Response:
def is_ipa(s):
    """Check if *s* consists of valid Chinese IPA."""
    re_pattern = ('(?:%(syllable)s|[ \t%(punctuation)s])+' %
                  {'syllable': _IPA_SYLLABLE,
                   'punctuation': zhon.pinyin.punctuation})
    return _is_pattern_match(re_pattern, s)


Instruction:
Create a Python 3 function for
identifying a given string s transcription system.

Response:
def identify(s):
    """Identify a given string's transcription system.

    *s* is the string to identify. The string is checked to see if its
    contents are valid Pinyin, Zhuyin, or IPA. The :data:`PINYIN`,
    :data:`ZHUYIN`, and :data:`IPA` constants are returned to indicate the
    string's identity.
    If *s* is not a valid transcription system, then :data:`UNKNOWN` is
    returned.

    When checking for valid Pinyin or Zhuyin, testing is done on a syllable
    level, not a character level. For example, just because a string is
    composed of characters used in Pinyin, doesn't mean that it will identify
    as Pinyin; it must actually consist of valid Pinyin syllables. The same
    applies for Zhuyin.

    When checking for IPA, testing is only done on a character level. In other
    words, a string just needs to consist of Chinese IPA characters in order
    to identify as IPA.

    """
    if is_pinyin(s):
        return PINYIN
    elif is_zhuyin(s):
        return ZHUYIN
    elif is_ipa(s):
        return IPA
    else:
        return UNKNOWN


Instruction:
Create a Python 3 function to
accept an objective function for optimization.

Response:
def prepare(self, f):
        """Accept an objective function for optimization."""
        self.g = autograd.grad(f)
        self.h = autograd.hessian(f)


Instruction:
Can you generate the documentation for the following Python 3 function
def optimize(self, x0, target):
        x = x0
        for _ in range(self.maxiter):
            delta = np.linalg.solve(self.h(x, target), -self.g(x, target))
            x = x + delta
            if np.linalg.norm(delta) < self.tol:
                break
        return x

Response: Calculate an optimum argument of an objective function.


Instruction:
Can you create a Python 3 function that
calculates an optimum argument of an objective function.

Response:
def optimize(self, x0, target):
        """Calculate an optimum argument of an objective function."""
        x = x0
        for i in range(self.maxiter):
            g = self.g(x, target)
            h = self.h(x, target)
            if i == 0:
                alpha = 0
                m = g
            else:
                alpha = - np.dot(m, np.dot(h, g)) / np.dot(m, np.dot(h, m))
                m = g + np.dot(alpha, m)
            t = - np.dot(m, g) / np.dot(m, np.dot(h, m))
            delta = np.dot(t, m)
            x = x + delta
            if np.linalg.norm(delta) < self.tol:
                break
        return x