Instruction:
Implement a Python 3 function for
generating a unique key for a BD2K URL.

Response:
def generate_unique_key(master_key_path, url):
    """
    Input1: Path to the BD2K Master Key (for S3 Encryption)
    Input2: S3 URL (e.g. https://s3-us-west-2.amazonaws.com/cgl-driver-projects-encrypted/wcdt/exome_bams/DTB-111-N.bam)

    Returns: 32-byte unique key generated for that URL
    """
    with open(master_key_path, 'r') as f:
        master_key = f.read()
    assert len(master_key) == 32, 'Invalid Key! Must be 32 characters. ' \
                                  'Key: {}, Length: {}'.format(master_key, len(master_key))
    new_key = hashlib.sha256(master_key + url).digest()
    assert len(new_key) == 32, 'New key is invalid and is not 32 characters: {}'.format(new_key)
    return new_key


Instruction:
Write a Python 3 function for
downloading encrypted file from S3 using the given key

Response:
def download_encrypted_file(work_dir, url, key_path, name):
    """
    Downloads encrypted file from S3

    Input1: Working directory
    Input2: S3 URL to be downloaded
    Input3: Path to key necessary for decryption
    Input4: name of file to be downloaded
    """
    file_path = os.path.join(work_dir, name)
    key = generate_unique_key(key_path, url)
    encoded_key = base64.b64encode(key)
    encoded_key_md5 = base64.b64encode(hashlib.md5(key).digest())
    h1 = 'x-amz-server-side-encryption-customer-algorithm:AES256'
    h2 = 'x-amz-server-side-encryption-customer-key:{}'.format(encoded_key)
    h3 = 'x-amz-server-side-encryption-customer-key-md5:{}'.format(encoded_key_md5)
    try:
        subprocess.check_call(['curl', '-fs', '--retry', '5', '-H', h1, '-H', h2, '-H', h3, url, '-o', file_path])
    except OSError:
        raise RuntimeError('Failed to find "curl". Install via "apt-get install curl"')
    assert os.path.exists(file_path)


Instruction:
Here you have a function in Python 3, explain what it does
def return_input_paths(job, work_dir, ids, *args):
    paths = OrderedDict()
    for name in args:
        if not os.path.exists(os.path.join(work_dir, name)):
            file_path = job.fileStore.readGlobalFile(ids[name], os.path.join(work_dir, name))
        else:
            file_path = os.path.join(work_dir, name)
        paths[name] = file_path
        if len(args) == 1:
            return file_path

    return paths.values()

Response: Returns the paths to the files requested by the FileStore


Instruction:
How would you code a function in Python 3 to
move files from work_dir to output_dir

Response:
def move_to_output_dir(work_dir, output_dir, uuid=None, files=list()):
    """
    Moves files from work_dir to output_dir

    Input1: Working directory
    Input2: Output directory
    Input3: UUID to be preprended onto file name
    Input4: list of file names to be moved from working dir to output dir
    """
    for fname in files:
        if uuid is None:
            shutil.move(os.path.join(work_dir, fname), os.path.join(output_dir, fname))
        else:
            shutil.move(os.path.join(work_dir, fname), os.path.join(output_dir, '{}.{}'.format(uuid, fname)))


Instruction:
Create a Python 3 function for
downloading shared files and places them in the jobstore.

Response:
def batch_start(job, input_args):
    """
    Downloads shared files that are used by all samples for alignment and places them in the jobstore.
    """
    shared_files = ['ref.fa', 'ref.fa.amb', 'ref.fa.ann', 'ref.fa.bwt', 'ref.fa.pac', 'ref.fa.sa', 'ref.fa.fai']
    shared_ids = {}
    for fname in shared_files:
        url = input_args[fname]
        shared_ids[fname] = job.addChildJobFn(download_from_url, url, fname).rv()
    job.addFollowOnJobFn(spawn_batch_jobs, shared_ids, input_args)


Instruction:
Write a Python 3 function that can
spawn an alignment job for every sample in the input configuration file

Response:
def spawn_batch_jobs(job, shared_ids, input_args):
    """
    Spawns an alignment job for every sample in the input configuration file
    """
    samples = []
    config = input_args['config']
    with open(config, 'r') as f_in:
        for line in f_in:
            line = line.strip().split(',')
            uuid = line[0]
            urls = line[1:]
            samples.append((uuid, urls))
    for sample in samples:
        job.addChildJobFn(alignment, shared_ids, input_args, sample, cores=32, memory='20 G', disk='100 G')


Instruction:
Write a Python 3 function for
running BWA and then Bamsort on the supplied fastqs for this sample Input1: Toil Job instance Input2: jobstore id dictionary Input3: Input arguments dictionary Input4: Sample tuple -- contains uuid and urls for the sample

Response:
def alignment(job, ids, input_args, sample):
    """
    Runs BWA and then Bamsort on the supplied fastqs for this sample

    Input1: Toil Job instance
    Input2: jobstore id dictionary
    Input3: Input arguments dictionary
    Input4: Sample tuple -- contains uuid and urls for the sample
    """
    uuid, urls = sample
    # ids['bam'] = job.fileStore.getEmptyFileStoreID()
    work_dir = job.fileStore.getLocalTempDir()
    output_dir = input_args['output_dir']
    key_path = input_args['ssec']
    cores = multiprocessing.cpu_count()

    # I/O
    return_input_paths(job, work_dir, ids, 'ref.fa', 'ref.fa.amb', 'ref.fa.ann',
                                                     'ref.fa.bwt', 'ref.fa.pac', 'ref.fa.sa', 'ref.fa.fai')
    # Get fastqs associated with this sample
    for url in urls:
        download_encrypted_file(work_dir, url, key_path, os.path.basename(url))

    # Parameters for BWA and Bamsort
    docker_cmd = ['docker', 'run', '--rm', '-v', '{}:/data'.format(work_dir)]

    bwa_command = ["jvivian/bwa",
                   "mem",
                   "-R", "@RG\tID:{0}\tPL:Illumina\tSM:{0}\tLB:KapaHyper".format(uuid),
                   "-T", str(0),
                   "-t", str(cores),
                   "/data/ref.fa"] + [os.path.join('/data/',  os.path.basename(x)) for x in urls]

    bamsort_command = ["jeltje/biobambam",
                       "/usr/local/bin/bamsort",
                       "inputformat=sam",
                       "level=1",
                       "inputthreads={}".format(cores),
                       "outputthreads={}".format(cores),
                       "calmdnm=1",
                       "calmdnmrecompindetonly=1",
                       "calmdnmreference=/data/ref.fa",
                       "I=/data/{}".format(uuid + '.sam')]
    # Piping the output to a file handle
    with open(os.path.join(work_dir, uuid + '.sam'), 'w') as f_out:
        subprocess.check_call(docker_cmd + bwa_command, stdout=f_out)

    with open(os.path.join(work_dir, uuid + '.bam'), 'w') as f_out:
        subprocess.check_call(docker_cmd + bamsort_command, stdout=f_out)

    # Save in JobStore
    # job.fileStore.updateGlobalFile(ids['bam'], os.path.join(work_dir, uuid + '.bam'))
    ids['bam'] = job.fileStore.writeGlobalFile(os.path.join(work_dir, uuid + '.bam'))
    # Copy file to S3
    if input_args['s3_dir']:
        job.addChildJobFn(upload_bam_to_s3, ids, input_args, sample, cores=32, memory='20 G', disk='30 G')
    # Move file in output_dir
    if input_args['output_dir']:
        move_to_output_dir(work_dir, output_dir, uuid=None, files=[uuid + '.bam'])


Instruction:
Given the following Python 3 function, write the documentation
def upload_bam_to_s3(job, ids, input_args, sample):
    uuid, urls = sample
    key_path = input_args['ssec']
    work_dir = job.fileStore.getLocalTempDir()
    # Parse s3_dir to get bucket and s3 path
    s3_dir = input_args['s3_dir']
    bucket_name = s3_dir.lstrip('/').split('/')[0]
    bucket_dir = '/'.join(s3_dir.lstrip('/').split('/')[1:])
    base_url = 'https://s3-us-west-2.amazonaws.com/'
    url = os.path.join(base_url, bucket_name, bucket_dir, uuid + '.bam')
    #I/O
    job.fileStore.readGlobalFile(ids['bam'], os.path.join(work_dir, uuid + '.bam'))
    # Generate keyfile for upload
    with open(os.path.join(work_dir, uuid + '.key'), 'wb') as f_out:
        f_out.write(generate_unique_key(key_path, url))
    # Commands to upload to S3 via S3AM
    s3am_command = ['s3am',
                    'upload',
                    '--sse-key-file', os.path.join(work_dir, uuid + '.key'),
                    'file://{}'.format(os.path.join(work_dir, uuid + '.bam')),
                    bucket_name,
                    os.path.join(bucket_dir, uuid + '.bam')]

    subprocess.check_call(s3am_command)

Response: Uploads BAM from sample to S3


Instruction:
Can you write a function in Python 3 where it
runs GATK Variant Quality Score Recalibration. 0: Start 0 --> 1 --> 3 --> 4 --> 5 1: Recalibrate SNPs | | 2: Recalibrate INDELS +-> 2 -+ 3: Apply SNP Recalibration 4: Apply INDEL Recalibration 5: Write VCF to output directory :param JobFunctionWrappingJob job: passed automatically by Toil :param str uuid: unique sample identifier :param str vcf_id: VCF FileStoreID :param Namespace config: Pipeline configuration options and shared files Requires the following config attributes: config.genome_fasta FilesStoreID for reference genome fasta file config.genome_fai FilesStoreID for reference genome fasta index file config.genome_dict FilesStoreID for reference genome sequence dictionary file config.cores Number of cores for each job config.xmx Java heap size in bytes config.suffix Suffix for output filename config.output_dir URL or local path to output directory config.ssec Path to key file for SSE-C encryption SNP VQSR attributes: config.snp_filter_annotations List of GATK variant annotations config.hapmap FileStoreID for HapMap resource file config.omni FileStoreID for Omni resource file config.dbsnp FileStoreID for dbSNP resource file config.g1k_snp FileStoreID for 1000G SNP resource file INDEL VQSR attributes: config.indel_filter_annotations List of GATK variant annotations config.dbsnp FileStoreID for dbSNP resource file config.mills FileStoreID for Mills resource file :return: SNP and INDEL VQSR VCF FileStoreID :rtype: str

Response:
def vqsr_pipeline(job, uuid, vcf_id, config):
    """
    Runs GATK Variant Quality Score Recalibration.

    0: Start                        0 --> 1 --> 3 --> 4 --> 5
    1: Recalibrate SNPs                   |      |
    2: Recalibrate INDELS                 +-> 2 -+
    3: Apply SNP Recalibration
    4: Apply INDEL Recalibration
    5: Write VCF to output directory

    :param JobFunctionWrappingJob job: passed automatically by Toil
    :param str uuid: unique sample identifier
    :param str vcf_id: VCF FileStoreID
    :param Namespace config: Pipeline configuration options and shared files
        Requires the following config attributes:
        config.genome_fasta             FilesStoreID for reference genome fasta file
        config.genome_fai               FilesStoreID for reference genome fasta index file
        config.genome_dict              FilesStoreID for reference genome sequence dictionary file
        config.cores                    Number of cores for each job
        config.xmx                      Java heap size in bytes
        config.suffix                   Suffix for output filename
        config.output_dir               URL or local path to output directory
        config.ssec                     Path to key file for SSE-C encryption

        SNP VQSR attributes:
        config.snp_filter_annotations   List of GATK variant annotations
        config.hapmap                   FileStoreID for HapMap resource file
        config.omni                     FileStoreID for Omni resource file
        config.dbsnp                    FileStoreID for dbSNP resource file
        config.g1k_snp                  FileStoreID for 1000G SNP resource file

        INDEL VQSR attributes:
        config.indel_filter_annotations List of GATK variant annotations
        config.dbsnp                    FileStoreID for dbSNP resource file
        config.mills                    FileStoreID for Mills resource file

    :return: SNP and INDEL VQSR VCF FileStoreID
    :rtype: str
    """
    # Get the total size of the genome reference
    genome_ref_size = config.genome_fasta.size + config.genome_fai.size + config.genome_dict.size

    # The VariantRecalibator disk requirement depends on the input VCF, the resource files,
    # the genome reference files, and the output recalibration table, tranche file, and plots.
    # The sum of these output files are less than the input VCF.
    snp_resources = ['hapmap', 'omni', 'dbsnp', 'g1k_snp']
    snp_resource_size = sum(getattr(config, resource).size for resource in snp_resources)
    snp_recal_disk = PromisedRequirement(lambda in_vcf, ref_size, resource_size:
                                         2 * in_vcf.size + ref_size + resource_size,
                                         vcf_id,
                                         genome_ref_size,
                                         snp_resource_size)

    snp_recal = job.wrapJobFn(gatk_variant_recalibrator,
                              'SNP',
                              vcf_id,
                              config.genome_fasta,
                              config.genome_fai,
                              config.genome_dict,
                              get_short_annotations(config.snp_filter_annotations),
                              hapmap=config.hapmap,
                              omni=config.omni,
                              phase=config.g1k_snp,
                              dbsnp=config.dbsnp,
                              unsafe_mode=config.unsafe_mode,
                              disk=snp_recal_disk,
                              cores=config.cores,
                              memory=config.xmx)

    indel_resource_size = config.mills.size + config.dbsnp.size
    indel_recal_disk = PromisedRequirement(lambda in_vcf, ref_size, resource_size:
                                           2 * in_vcf.size + ref_size + resource_size,
                                           vcf_id,
                                           genome_ref_size,
                                           indel_resource_size)

    indel_recal = job.wrapJobFn(gatk_variant_recalibrator,
                                'INDEL',
                                vcf_id,
                                config.genome_fasta,
                                config.genome_fai,
                                config.genome_dict,
                                get_short_annotations(config.indel_filter_annotations),
                                dbsnp=config.dbsnp,
                                mills=config.mills,
                                unsafe_mode=config.unsafe_mode,
                                disk=indel_recal_disk,
                                cores=config.cores,
                                memory=config.xmx)

    # The ApplyRecalibration disk requirement depends on the input VCF size, the variant
    # recalibration table, the tranche file, the genome reference file, and the output VCF.
    # This step labels variants as filtered, so the output VCF file should be slightly larger
    # than the input file. Estimate a 10% increase in the VCF file size.
    apply_snp_recal_disk = PromisedRequirement(lambda in_vcf, recal, tranche, ref_size:
                                               int(2.1 * in_vcf.size + recal.size + tranche.size + ref_size),
                                               vcf_id,
                                               snp_recal.rv(0),
                                               snp_recal.rv(1),
                                               genome_ref_size)

    apply_snp_recal = job.wrapJobFn(gatk_apply_variant_recalibration,
                                    'SNP',
                                    vcf_id,
                                    snp_recal.rv(0), snp_recal.rv(1),
                                    config.genome_fasta,
                                    config.genome_fai,
                                    config.genome_dict,
                                    unsafe_mode=config.unsafe_mode,
                                    disk=apply_snp_recal_disk,
                                    cores=config.cores,
                                    memory=config.xmx)

    apply_indel_recal_disk = PromisedRequirement(lambda in_vcf, recal, tranche, ref_size:
                                                 int(2.1 * in_vcf.size + recal.size + tranche.size + ref_size),
                                                 vcf_id,
                                                 indel_recal.rv(0),
                                                 indel_recal.rv(1),
                                                 genome_ref_size)

    apply_indel_recal = job.wrapJobFn(gatk_apply_variant_recalibration,
                                      'INDEL',
                                      apply_snp_recal.rv(),
                                      indel_recal.rv(0), indel_recal.rv(1),
                                      config.genome_fasta,
                                      config.genome_fai,
                                      config.genome_dict,
                                      unsafe_mode=config.unsafe_mode,
                                      disk=apply_indel_recal_disk,
                                      cores=config.cores,
                                      memory=config.xmx)

    job.addChild(snp_recal)
    job.addChild(indel_recal)
    snp_recal.addChild(apply_snp_recal)
    indel_recal.addChild(apply_indel_recal)
    apply_snp_recal.addChild(apply_indel_recal)

    # Output recalibrated VCF
    output_dir = config.output_dir
    output_dir = os.path.join(output_dir, uuid)
    vqsr_name = '%s.vqsr%s.vcf' % (uuid, config.suffix)
    output_vqsr = job.wrapJobFn(output_file_job,
                                vqsr_name,
                                apply_indel_recal.rv(),
                                output_dir,
                                s3_key_path=config.ssec,
                                disk=PromisedRequirement(lambda x: x.size, apply_indel_recal.rv()))
    apply_indel_recal.addChild(output_vqsr)
    return apply_indel_recal.rv()


Instruction:
Write a Python 3 script to
convert full GATK annotation name to the shortened version of the GATK annotation name.

Response:
def get_short_annotations(annotations):
    """
    Converts full GATK annotation name to the shortened version
    :param annotations:
    :return:
    """
    # Annotations need to match VCF header
    short_name = {'QualByDepth': 'QD',
                  'FisherStrand': 'FS',
                  'StrandOddsRatio': 'SOR',
                  'ReadPosRankSumTest': 'ReadPosRankSum',
                  'MappingQualityRankSumTest': 'MQRankSum',
                  'RMSMappingQuality': 'MQ',
                  'InbreedingCoeff': 'ID'}

    short_annotations = []
    for annotation in annotations:
        if annotation in short_name:
            annotation = short_name[annotation]
        short_annotations.append(annotation)
    return short_annotations


Instruction:
Make a summary of the following Python 3 code
def parse_sra(path_to_config):
    samples = []
    with open(path_to_config, 'r') as f:
        for line in f.readlines():
            if not line.isspace():
                samples.append(line.strip())
    return samples

Response: Parses genetorrent config file. Returns list of samples.


Instruction:
Implement a Python 3 function for
taring a group of files together into a tarball

Response:
def tarball_files(work_dir, tar_name, uuid=None, files=None):
    """
    Tars a group of files together into a tarball

    work_dir: str       Current Working Directory
    tar_name: str       Name of tarball
    uuid: str           UUID to stamp files with
    files: str(s)       List of filenames to place in the tarball from working directory
    """
    with tarfile.open(os.path.join(work_dir, tar_name), 'w:gz') as f_out:
        for fname in files:
            if uuid:
                f_out.add(os.path.join(work_dir, fname), arcname=uuid + '.' + fname)
            else:
                f_out.add(os.path.join(work_dir, fname), arcname=fname)


Instruction:
Can you create a Python 3 function that
downloads a sample from dbGaP then uses S3AM to transfer it to CGHub

Response:
def download_and_transfer_sample(job, input_args, samples):
    """
    Downloads a sample from dbGaP via SRAToolKit, then uses S3AM to transfer it to S3

    input_args: dict        Dictionary of input arguments
    analysis_id: str        An analysis ID for a sample in CGHub
    """
    if len(samples) > 1:
        a = samples[len(samples)/2:]
        b = samples[:len(samples)/2]
        job.addChildJobFn(download_and_transfer_sample, input_args, a, disk='30G')
        job.addChildJobFn(download_and_transfer_sample, input_args, b, disk='30G')
    else:
        analysis_id = samples[0]
        work_dir = job.fileStore.getLocalTempDir()
        sudo = input_args['sudo']
        # Acquire dbgap_key
        shutil.copy(input_args['dbgap_key'], os.path.join(work_dir, 'dbgap.ngc'))
        # Call to fastq-dump to pull down SRA files and convert to fastq
        if input_args['single_end']:
            parameters = [analysis_id]
        else:
            parameters = ['--split-files', analysis_id]
        docker_call(tool='quay.io/ucsc_cgl/fastq-dump:2.5.7--4577a6c1a3c94adaa0c25dd6c03518ee610433d1',
                    work_dir=work_dir, tool_parameters=parameters, sudo=sudo)
        # Collect files and encapsulate into a tarball
        shutil.rmtree(os.path.join(work_dir, 'sra'))
        sample_name = analysis_id + '.tar.gz'
        if input_args['single_end']:
            r = [os.path.basename(x) for x in glob.glob(os.path.join(work_dir, '*.f*'))]
            tarball_files(work_dir, tar_name=sample_name, files=r)
        else:
            r1 = [os.path.basename(x) for x in glob.glob(os.path.join(work_dir, '*_1*'))]
            r2 = [os.path.basename(x) for x in glob.glob(os.path.join(work_dir, '*_2*'))]
            tarball_files(work_dir, tar_name=sample_name, files=r1 + r2)
        # Parse s3_dir to get bucket and s3 path
        key_path = input_args['ssec']
        s3_dir = input_args['s3_dir']
        bucket_name = s3_dir.lstrip('/').split('/')[0]
        base_url = 'https://s3-us-west-2.amazonaws.com/'
        url = os.path.join(base_url, bucket_name, sample_name)
        # Generate keyfile for upload
        with open(os.path.join(work_dir, 'temp.key'), 'wb') as f_out:
            f_out.write(generate_unique_key(key_path, url))
        # Upload to S3 via S3AM
        s3am_command = ['s3am',
                        'upload',
                        '--sse-key-file', os.path.join(work_dir, 'temp.key'),
                        'file://{}'.format(os.path.join(work_dir, sample_name)),
                        's3://' + bucket_name + '/']
        subprocess.check_call(s3am_command)


Instruction:
Write a Python 3 script for
transferring gTEX data from dbGaP (NCBI) to S3

Response:
def main():
    """
    Transfer gTEX data from dbGaP (NCBI) to S3
    """
    # Define Parser object and add to toil
    parser = build_parser()
    Job.Runner.addToilOptions(parser)
    args = parser.parse_args()
    # Store inputs from argparse
    inputs = {'sra': args.sra,
              'dbgap_key': args.dbgap_key,
              'ssec': args.ssec,
              's3_dir': args.s3_dir,
              'single_end': args.single_end,
              'sudo': args.sudo}
    # Sanity checks
    if args.ssec:
        assert os.path.isfile(args.ssec)
    if args.sra:
        assert os.path.isfile(args.sra)
    if args.dbgap_key:
        assert os.path.isfile(args.dbgap_key)
    # Start Pipeline
    Job.Runner.startToil(Job.wrapJobFn(start_batch, inputs), args)


Instruction:
Can you generate the documentation for the following Python 3 function
def output_file_job(job, filename, file_id, output_dir, s3_key_path=None):
    job.fileStore.logToMaster('Writing {} to {}'.format(filename, output_dir))
    work_dir = job.fileStore.getLocalTempDir()
    filepath = job.fileStore.readGlobalFile(file_id, os.path.join(work_dir, filename))
    if urlparse(output_dir).scheme == 's3':
        s3am_upload(job=job, fpath=os.path.join(work_dir, filepath),
                    s3_dir=output_dir,
                    s3_key_path=s3_key_path)
    elif os.path.exists(os.path.join(output_dir, filename)):
        job.fileStore.logToMaster("File already exists: {}".format(filename))
    else:
        mkdir_p(output_dir)
        copy_files([filepath], output_dir)

Response: Uploads a file from the FileStore to an output directory on the local filesystem or S3.


Instruction:
Create a Python 3 function to
download encrypted files from S3 via header injection

Response:
def download_encrypted_file(job, input_args, name):
    """
    Downloads encrypted files from S3 via header injection

    input_args: dict    Input dictionary defined in main()
    name: str           Symbolic name associated with file
    """
    work_dir = job.fileStore.getLocalTempDir()
    key_path = input_args['ssec']
    file_path = os.path.join(work_dir, name)
    url = input_args[name]

    with open(key_path, 'r') as f:
        key = f.read()
    if len(key) != 32:
        raise RuntimeError('Invalid Key! Must be 32 bytes: {}'.format(key))

    key = generate_unique_key(key_path, url)

    encoded_key = base64.b64encode(key)
    encoded_key_md5 = base64.b64encode(hashlib.md5(key).digest())
    h1 = 'x-amz-server-side-encryption-customer-algorithm:AES256'
    h2 = 'x-amz-server-side-encryption-customer-key:{}'.format(encoded_key)
    h3 = 'x-amz-server-side-encryption-customer-key-md5:{}'.format(encoded_key_md5)
    try:
        subprocess.check_call(['curl', '-fs', '--retry', '5', '-H', h1, '-H', h2, '-H', h3, url, '-o', file_path])
    except OSError:
        raise RuntimeError('Failed to find "curl". Install via "apt-get install curl"')
    assert os.path.exists(file_path)
    return job.fileStore.writeGlobalFile(file_path)


Instruction:
Write a Python 3 function that can
download a single file from a given URL.

Response:
def download_from_url(job, url):
    """
    Simple curl request made for a given url

    url: str    URL to download
    """
    work_dir = job.fileStore.getLocalTempDir()
    file_path = os.path.join(work_dir, os.path.basename(url))
    if not os.path.exists(file_path):
        if url.startswith('s3:'):
            download_from_s3_url(file_path, url)
        else:
            try:
                subprocess.check_call(['curl', '-fs', '--retry', '5', '--create-dir', url, '-o', file_path])
            except OSError:
                raise RuntimeError('Failed to find "curl". Install via "apt-get install curl"')
    assert os.path.exists(file_path)
    return job.fileStore.writeGlobalFile(file_path)


Instruction:
Can you generate the documentation for the following Python 3 function
def docker_call(work_dir, tool_parameters, tool, java_opts=None, outfile=None, sudo=False):
    base_docker_call = 'docker run --log-driver=none --rm -v {}:/data'.format(work_dir).split()
    if sudo:
        base_docker_call = ['sudo'] + base_docker_call
    if java_opts:
        base_docker_call = base_docker_call + ['-e', 'JAVA_OPTS={}'.format(java_opts)]
    try:
        if outfile:
            subprocess.check_call(base_docker_call + [tool] + tool_parameters, stdout=outfile)
        else:
            subprocess.check_call(base_docker_call + [tool] + tool_parameters)
    except subprocess.CalledProcessError:
        raise RuntimeError('docker command returned a non-zero exit status. Check error logs.')
    except OSError:
        raise RuntimeError('docker not found on system. Install on all nodes.')

Response: Makes subprocess call of a command to a docker container.


Instruction:
Write a Python 3 function that can
copy files from work_dir to output_dir.

Response:
def copy_to_output_dir(work_dir, output_dir, uuid=None, files=list()):
    """
    A list of files to move from work_dir to output_dir.

    work_dir: str       Current working directory
    output_dir: str     Output directory for files to go
    uuid: str           UUID to "stamp" onto output files
    files: list         List of files to iterate through
    """
    for fname in files:
        if uuid is None:
            shutil.copy(os.path.join(work_dir, fname), os.path.join(output_dir, fname))
        else:
            shutil.copy(os.path.join(work_dir, fname), os.path.join(output_dir, '{}.{}'.format(uuid, fname)))


Instruction:
Create a Python 3 function for
checking that dependency programs are installed on every node.

Response:
def program_checks(job, input_args):
    """
    Checks that dependency programs are installed.

    input_args: dict        Dictionary of input arguments (from main())
    """
    # Program checks
    for program in ['curl', 'docker', 'unzip', 'samtools']:
        assert which(program), 'Program "{}" must be installed on every node.'.format(program)
    job.addChildJobFn(download_shared_files, input_args)


Instruction:
Make a summary of the following Python 3 code
def download_shared_files(job, input_args):
    shared_files = ['unc.bed', 'hg19.transcripts.fa', 'composite_exons.bed', 'normalize.pl', 'rsem_ref.zip',
                    'ebwt.zip', 'chromosomes.zip']
    shared_ids = {}
    for f in shared_files:
        shared_ids[f] = job.addChildJobFn(download_from_url, input_args[f]).rv()
    if input_args['config'] or input_args['config_fastq']:
        job.addFollowOnJobFn(parse_config_file, shared_ids, input_args)
    else:
        sample_path = input_args['input']
        uuid = os.path.splitext(os.path.basename(sample_path))[0]
        sample = (uuid, sample_path)
        job.addFollowOnJobFn(download_sample, shared_ids, input_args, sample)

Response: Downloads and stores shared files in the FileStore.


Instruction:
Can you implement a function in Python 3 that
parses the config file and adds the pipeline for each sample.

Response:
def parse_config_file(job, ids, input_args):
    """
    Launches pipeline for each sample.

    shared_ids: dict        Dictionary of fileStore IDs
    input_args: dict        Dictionary of input arguments
    """
    samples = []
    config = input_args['config']
    with open(config, 'r') as f:
        for line in f.readlines():
            if not line.isspace():
                sample = line.strip().split(',')
                samples.append(sample)
    for sample in samples:
        job.addChildJobFn(download_sample, ids, input_args, sample)


Instruction:
Explain what the following Python 3 code does
def download_sample(job, ids, input_args, sample):
    if len(sample) == 2:
        uuid, sample_location = sample
        url1, url2 = None, None
    else:
        uuid, url1, url2 = sample
        sample_location = None
    # Update values unique to sample
    sample_input = dict(input_args)
    sample_input['uuid'] = uuid
    sample_input['sample.tar'] = sample_location
    if sample_input['output_dir']:
        sample_input['output_dir'] = os.path.join(input_args['output_dir'], uuid)
    sample_input['cpu_count'] = multiprocessing.cpu_count()
    job_vars = (sample_input, ids)
    # Download or locate local file and place in the jobStore
    if sample_input['input']:
        ids['sample.tar'] = job.fileStore.writeGlobalFile(os.path.abspath(sample_location))
    elif sample_input['config_fastq']:
        ids['R1.fastq'] = job.fileStore.writeGlobalFile(urlparse(url1).path)
        ids['R2.fastq'] = job.fileStore.writeGlobalFile(urlparse(url2).path)
    else:
        if sample_input['ssec']:
            ids['sample.tar'] = job.addChildJobFn(download_encrypted_file, sample_input, 'sample.tar', disk='25G').rv()
        else:
            ids['sample.tar'] = job.addChildJobFn(download_from_url, sample_input['sample.tar'], disk='25G').rv()
    job.addFollowOnJobFn(static_dag_launchpoint, job_vars)

Response: Download a sample from the file store and place it in the jobStore.


Instruction:
How would you explain what the following Python 3 function does
def merge_fastqs(job, job_vars):
    input_args, ids = job_vars
    work_dir = job.fileStore.getLocalTempDir()
    cores = input_args['cpu_count']
    single_end_reads = input_args['single_end_reads']
    # I/O
    sample = return_input_paths(job, work_dir, ids, 'sample.tar')
    # Untar File
    # subprocess.check_call(['unzip', sample, '-d', work_dir])
    subprocess.check_call(['tar', '-xvf', sample, '-C', work_dir])
    # Remove large files before creating concat versions.
    os.remove(os.path.join(work_dir, 'sample.tar'))
    # Zcat files in parallel
    if single_end_reads:
        files = sorted(glob.glob(os.path.join(work_dir, '*')))
        with open(os.path.join(work_dir, 'R1.fastq'), 'w') as f1:
            subprocess.check_call(['zcat'] + files, stdout=f1)
        # FileStore
        ids['R1.fastq'] = job.fileStore.writeGlobalFile(os.path.join(work_dir, 'R1.fastq'))
    else:
        r1_files = sorted(glob.glob(os.path.join(work_dir, '*R1*')))
        r2_files = sorted(glob.glob(os.path.join(work_dir, '*R2*')))
        with open(os.path.join(work_dir, 'R1.fastq'), 'w') as f1:
            p1 = subprocess.Popen(['zcat'] + r1_files, stdout=f1)
        with open(os.path.join(work_dir, 'R2.fastq'), 'w') as f2:
            p2 = subprocess.Popen(['zcat'] + r2_files, stdout=f2)
        p1.wait()
        p2.wait()
        # FileStore
        ids['R1.fastq'] = job.fileStore.writeGlobalFile(os.path.join(work_dir, 'R1.fastq'))
        ids['R2.fastq'] = job.fileStore.writeGlobalFile(os.path.join(work_dir, 'R2.fastq'))
    job.fileStore.deleteGlobalFile(ids['sample.tar'])
    # Spawn child job
    return job.addChildJobFn(mapsplice, job_vars, cores=cores, disk='130 G').rv()

Response: Merge the fastqs of the Read1 and Read2 groups together.


Instruction:
Can you tell what is the following Python 3 function doing
def mapsplice(job, job_vars):
    # Unpack variables
    input_args, ids = job_vars
    work_dir = job.fileStore.getLocalTempDir()
    cores = input_args['cpu_count']
    sudo = input_args['sudo']
    single_end_reads = input_args['single_end_reads']
    files_to_delete = ['R1.fastq']
    # I/O
    return_input_paths(job, work_dir, ids, 'ebwt.zip', 'chromosomes.zip')
    if single_end_reads:
        return_input_paths(job, work_dir, ids, 'R1.fastq')
    else:
        return_input_paths(job, work_dir, ids, 'R1.fastq', 'R2.fastq')
        files_to_delete.extend(['R2.fastq'])
    for fname in ['chromosomes.zip', 'ebwt.zip']:
        subprocess.check_call(['unzip', '-o', os.path.join(work_dir, fname), '-d', work_dir])
    # Command and call
    parameters = ['-p', str(cores),
                  '-s', '25',
                  '--bam',
                  '--min-map-len', '50',
                  '-x', '/data/ebwt',
                  '-c', '/data/chromosomes',
                  '-1', '/data/R1.fastq',
                  '-o', '/data']
    if not single_end_reads:
        parameters.extend(['-2', '/data/R2.fastq'])
    docker_call(tool='quay.io/ucsc_cgl/mapsplice:2.1.8--dd5ac549b95eb3e5d166a5e310417ef13651994e',
                tool_parameters=parameters, work_dir=work_dir, sudo=sudo)
    # Write to FileStore
    for fname in ['alignments.bam', 'stats.txt']:
        ids[fname] = job.fileStore.writeGlobalFile(os.path.join(work_dir, fname))
    for fname in files_to_delete:
        job.fileStore.deleteGlobalFile(ids[fname])
    # Run child job
    # map_id = job.addChildJobFn(mapping_stats, job_vars).rv()
    if input_args['upload_bam_to_s3'] and input_args['s3_dir']:
        job.addChildJobFn(upload_bam_to_s3, job_vars)
    output_ids = job.addChildJobFn(add_read_groups, job_vars, disk='30 G').rv()
    return output_ids

Response: Maps RNA - Seq reads to a reference genome.


Instruction:
Can you tell what is the following Python 3 function doing
def add_read_groups(job, job_vars):
    input_args, ids = job_vars
    work_dir = job.fileStore.getLocalTempDir()
    sudo = input_args['sudo']
    # I/O
    alignments = return_input_paths(job, work_dir, ids, 'alignments.bam')
    output = os.path.join(work_dir, 'rg_alignments.bam')
    # Command and callg
    parameter = ['AddOrReplaceReadGroups',
                 'INPUT={}'.format(docker_path(alignments)),
                 'OUTPUT={}'.format(docker_path(output)),
                 'RGSM={}'.format(input_args['uuid']),
                 'RGID={}'.format(input_args['uuid']),
                 'RGLB=TruSeq',
                 'RGPL=illumina',
                 'RGPU=barcode',
                 'VALIDATION_STRINGENCY=SILENT']
    docker_call(tool='quay.io/ucsc_cgl/picardtools:1.95--dd5ac549b95eb3e5d166a5e310417ef13651994e',
                tool_parameters=parameter, work_dir=work_dir, sudo=sudo)
    # Write to FileStore
    ids['rg_alignments.bam'] = job.fileStore.writeGlobalFile(output)
    # Run child job
    return job.addChildJobFn(bamsort_and_index, job_vars, disk='30 G').rv()

Response: This function adds read groups to the headers


Instruction:
Implement a function in Python 3 to
sort bam file and produces index file

Response:
def bamsort_and_index(job, job_vars):
    """
    Sorts bam file and produces index file

    job_vars: tuple     Tuple of dictionaries: input_args and ids
    """
    # Unpack variables
    input_args, ids = job_vars
    work_dir = job.fileStore.getLocalTempDir()
    sudo = input_args['sudo']
    # I/O
    rg_alignments = return_input_paths(job, work_dir, ids, 'rg_alignments.bam')
    output = os.path.join(work_dir, 'sorted.bam')
    # Command -- second argument is "Output Prefix"
    cmd1 = ['sort', docker_path(rg_alignments), docker_path('sorted')]
    cmd2 = ['index', docker_path(output)]
    docker_call(tool='quay.io/ucsc_cgl/samtools:0.1.19--dd5ac549b95eb3e5d166a5e310417ef13651994e',
                tool_parameters=cmd1, work_dir=work_dir, sudo=sudo)
    docker_call(tool='quay.io/ucsc_cgl/samtools:0.1.19--dd5ac549b95eb3e5d166a5e310417ef13651994e',
                tool_parameters=cmd2, work_dir=work_dir, sudo=sudo)
    # Write to FileStore
    ids['sorted.bam'] = job.fileStore.writeGlobalFile(output)
    ids['sorted.bam.bai'] = job.fileStore.writeGlobalFile(os.path.join(work_dir, 'sorted.bam.bai'))
    # Run child job
    output_ids = job.addChildJobFn(sort_bam_by_reference, job_vars, disk='50 G').rv()
    rseq_id = job.addChildJobFn(rseq_qc, job_vars, disk='20 G').rv()
    return rseq_id, output_ids


Instruction:
Here you have a function in Python 3, explain what it does
def rseq_qc(job, job_vars):
    input_args, ids = job_vars
    work_dir = job.fileStore.getLocalTempDir()
    uuid = input_args['uuid']
    sudo = input_args['sudo']
    # I/O
    return_input_paths(job, work_dir, ids, 'sorted.bam', 'sorted.bam.bai')
    # Command
    docker_call(tool='jvivian/qc', tool_parameters=['/opt/cgl-docker-lib/RseqQC_v2.sh', '/data/sorted.bam', uuid],
                work_dir=work_dir, sudo=sudo)
    # Write to FileStore
    output_files = [f for f in glob.glob(os.path.join(work_dir, '*')) if 'sorted.bam' not in f]
    tarball_files(work_dir, tar_name='qc.tar.gz', uuid=None, files=output_files)
    return job.fileStore.writeGlobalFile(os.path.join(work_dir, 'qc.tar.gz'))

Response: This module is used to run RseqQC on the BAM post alignment. It is used to run RseqQC on the BAM post alignment.


Instruction:
Write a Python 3 function for
sorting the bam by reference

Response:
def sort_bam_by_reference(job, job_vars):
    """
    Sorts the bam by reference

    job_vars: tuple     Tuple of dictionaries: input_args and ids
    """
    # Unpack variables
    input_args, ids = job_vars
    work_dir = job.fileStore.getLocalTempDir()
    # I/O
    sorted_bam, sorted_bai = return_input_paths(job, work_dir, ids, 'sorted.bam', 'sorted.bam.bai')
    output = os.path.join(work_dir, 'sort_by_ref.bam')
    # Call: Samtools
    ref_seqs = []
    handle = subprocess.Popen(["samtools", "view", "-H", sorted_bam], stdout=subprocess.PIPE).stdout
    for line in handle:
        if line.startswith("@SQ"):
            tmp = line.split("\t")
            chrom = tmp[1].split(":")[1]
            ref_seqs.append(chrom)
    handle.close()
    # Iterate through chromosomes to create mini-bams
    for chrom in ref_seqs:
        # job.addChildJobFn(sbbr_child, chrom, os.path.join(work_dir, chrom), sorted_bam)
        cmd_view = ["samtools", "view", "-b", sorted_bam, chrom]
        cmd_sort = ["samtools", "sort", "-m", "3000000000", "-n", "-", os.path.join(work_dir, chrom)]
        p1 = subprocess.Popen(cmd_view, stdout=subprocess.PIPE)
        subprocess.check_call(cmd_sort, stdin=p1.stdout)
    sorted_files = [os.path.join(work_dir, chrom) + '.bam' for chrom in ref_seqs]
    cmd = ["samtools", "cat", "-o", output] + sorted_files
    subprocess.check_call(cmd)
    # Write to FileStore
    ids['sort_by_ref.bam'] = job.fileStore.writeGlobalFile(output)
    rsem_id = job.addChildJobFn(transcriptome, job_vars, disk='30 G', memory='30 G').rv()
    exon_id = job.addChildJobFn(exon_count, job_vars, disk='30 G').rv()
    return exon_id, rsem_id


Instruction:
How would you explain what the following Python 3 function does
def exon_count(job, job_vars):
    input_args, ids = job_vars
    work_dir = job.fileStore.getLocalTempDir()
    uuid = input_args['uuid']
    sudo = input_args['sudo']
    # I/O
    sort_by_ref, normalize_pl, composite_bed = return_input_paths(job, work_dir, ids, 'sort_by_ref.bam',
                                                                  'normalize.pl', 'composite_exons.bed')
    # Command
    tool = 'jvivian/bedtools'
    cmd_1 = ['coverage',
             '-split',
             '-abam', docker_path(sort_by_ref),
             '-b', docker_path(composite_bed)]
    cmd_2 = ['perl',
             os.path.join(work_dir, 'normalize.pl'),
             sort_by_ref,
             composite_bed]

    popen_docker = ['docker', 'run', '-v', '{}:/data'.format(work_dir), tool]
    if sudo:
        popen_docker = ['sudo'] + popen_docker
    p = subprocess.Popen(popen_docker + cmd_1, stdout=subprocess.PIPE)
    with open(os.path.join(work_dir, 'exon_quant'), 'w') as f:
        subprocess.check_call(cmd_2, stdin=p.stdout, stdout=f)

    p1 = subprocess.Popen(['cat', os.path.join(work_dir, 'exon_quant')], stdout=subprocess.PIPE)
    p2 = subprocess.Popen(['tr', '":"', '"\t"'], stdin=p1.stdout, stdout=subprocess.PIPE)
    p3 = subprocess.Popen(['tr', '"-"', '"\t"'], stdin=p2.stdout, stdout=subprocess.PIPE)
    with open(os.path.join(work_dir, 'exon_quant.bed'), 'w') as f:
        subprocess.check_call(['cut', '-f1-4'], stdin=p3.stdout, stdout=f)
    # Create zip, upload to fileStore, and move to output_dir as a backup
    output_files = ['exon_quant.bed', 'exon_quant']
    tarball_files(work_dir, tar_name='exon.tar.gz', uuid=uuid, files=output_files)
    return job.fileStore.writeGlobalFile(os.path.join(work_dir, 'exon.tar.gz'))

Response: Generates exon counts for a given set of exons.


Instruction:
How would you code a function in Python 3 to
create a bam of just the transcriptome job_vars: tuple Tuple of dictionaries: input_args and ids

Response:
def transcriptome(job, job_vars):
    """
    Creates a bam of just the transcriptome

    job_vars: tuple     Tuple of dictionaries: input_args and ids
    """
    input_args, ids = job_vars
    work_dir = job.fileStore.getLocalTempDir()
    sudo = input_args['sudo']
    # I/O
    sort_by_ref, bed, hg19_fa = return_input_paths(job, work_dir, ids, 'sort_by_ref.bam',
                                                   'unc.bed', 'hg19.transcripts.fa')
    output = os.path.join(work_dir, 'transcriptome.bam')
    # Command
    parameters = ['sam-xlate',
                  '--bed', docker_path(bed),
                  '--in', docker_path(sort_by_ref),
                  '--order', docker_path(hg19_fa),
                  '--out', docker_path(output),
                  '--xgtag',
                  '--reverse']
    docker_call(tool='quay.io/ucsc_cgl/ubu:1.2--02806964cdf74bf5c39411b236b4c4e36d026843',
                tool_parameters=parameters, work_dir=work_dir, java_opts='-Xmx30g', sudo=sudo)
    # Write to FileStore
    ids['transcriptome.bam'] = job.fileStore.writeGlobalFile(output)
    # Run child job
    return job.addChildJobFn(filter_bam, job_vars, memory='30G', disk='30G').rv()


Instruction:
Write a Python 3 script to
perform filtering on the transcriptome bam job_vars: tuple Tuple of dictionaries: input_args and ids

Response:
def filter_bam(job, job_vars):
    """
    Performs filtering on the transcriptome bam

    job_vars: tuple     Tuple of dictionaries: input_args and ids
    """
    input_args, ids = job_vars
    work_dir = job.fileStore.getLocalTempDir()
    cores = input_args['cpu_count']
    sudo = input_args['sudo']
    # I/O
    transcriptome_bam = return_input_paths(job, work_dir, ids, 'transcriptome.bam')
    output = os.path.join(work_dir, 'filtered.bam')
    # Command
    parameters = ['sam-filter',
                  '--strip-indels',
                  '--max-insert', '1000',
                  '--mapq', '1',
                  '--in', docker_path(transcriptome_bam),
                  '--out', docker_path(output)]
    docker_call(tool='quay.io/ucsc_cgl/ubu:1.2--02806964cdf74bf5c39411b236b4c4e36d026843',
                tool_parameters=parameters, work_dir=os.path.dirname(output), java_opts='-Xmx30g', sudo=sudo)
    # Write to FileStore
    ids['filtered.bam'] = job.fileStore.writeGlobalFile(output)
    # Run child job
    return job.addChildJobFn(rsem, job_vars, cores=cores, disk='30 G').rv()


Instruction:
Create a Python 3 function to
run RSEM to produce counts

Response:
def rsem(job, job_vars):
    """
    Runs RSEM to produce counts

    job_vars: tuple     Tuple of dictionaries: input_args and ids
    """
    input_args, ids = job_vars
    work_dir = job.fileStore.getLocalTempDir()
    cpus = input_args['cpu_count']
    sudo = input_args['sudo']
    single_end_reads = input_args['single_end_reads']
    # I/O
    filtered_bam, rsem_ref = return_input_paths(job, work_dir, ids, 'filtered.bam', 'rsem_ref.zip')
    subprocess.check_call(['unzip', '-o', os.path.join(work_dir, 'rsem_ref.zip'), '-d', work_dir])
    output_prefix = 'rsem'
    # Make tool call to Docker
    parameters = ['--quiet',
                  '--no-qualities',
                  '-p', str(cpus),
                  '--forward-prob', '0.5',
                  '--seed-length', '25',
                  '--fragment-length-mean', '-1.0',
                  '--bam', docker_path(filtered_bam)]
    if not single_end_reads:
        parameters.extend(['--paired-end'])
    parameters.extend(['/data/rsem_ref/hg19_M_rCRS_ref', output_prefix])

    docker_call(tool='quay.io/ucsc_cgl/rsem:1.2.25--4e8d1b31d4028f464b3409c6558fb9dfcad73f88',
                tool_parameters=parameters, work_dir=work_dir, sudo=sudo)
    os.rename(os.path.join(work_dir, output_prefix + '.genes.results'), os.path.join(work_dir, 'rsem_gene.tab'))
    os.rename(os.path.join(work_dir, output_prefix + '.isoforms.results'), os.path.join(work_dir, 'rsem_isoform.tab'))
    # Write to FileStore
    ids['rsem_gene.tab'] = job.fileStore.writeGlobalFile(os.path.join(work_dir, 'rsem_gene.tab'))
    ids['rsem_isoform.tab'] = job.fileStore.writeGlobalFile(os.path.join(work_dir, 'rsem_isoform.tab'))
    # Run child jobs
    return job.addChildJobFn(rsem_postprocess, job_vars).rv()


Instruction:
Make a summary of the following Python 3 code
def consolidate_output(job, job_vars, output_ids):
    input_args, ids = job_vars
    work_dir = job.fileStore.getLocalTempDir()
    uuid = input_args['uuid']
    # Retrieve IDs
    rseq_id, exon_id, rsem_id = flatten(output_ids)
    # Retrieve output file paths to consolidate
    # map_tar = job.fileStore.readGlobalFile(map_id, os.path.join(work_dir, 'map.tar.gz'))
    qc_tar = job.fileStore.readGlobalFile(rseq_id, os.path.join(work_dir, 'qc.tar.gz'))
    exon_tar = job.fileStore.readGlobalFile(exon_id, os.path.join(work_dir, 'exon.tar.gz'))
    rsem_tar = job.fileStore.readGlobalFile(rsem_id, os.path.join(work_dir, 'rsem.tar.gz'))
    # I/O
    out_tar = os.path.join(work_dir, uuid + '.tar.gz')
    # Consolidate separate tarballs
    with tarfile.open(os.path.join(work_dir, out_tar), 'w:gz') as f_out:
        for tar in [rsem_tar, exon_tar, qc_tar]:
            with tarfile.open(tar, 'r') as f_in:
                for tarinfo in f_in:
                    with closing(f_in.extractfile(tarinfo)) as f_in_file:
                        if tar == qc_tar:
                            tarinfo.name = os.path.join(uuid, 'rseq_qc', os.path.basename(tarinfo.name))
                        else:
                            tarinfo.name = os.path.join(uuid, os.path.basename(tarinfo.name))
                        f_out.addfile(tarinfo, fileobj=f_in_file)
    # Move to output directory of selected
    if input_args['output_dir']:
        output_dir = input_args['output_dir']
        mkdir_p(output_dir)
        copy_to_output_dir(work_dir, output_dir, uuid=None, files=[uuid + '.tar.gz'])
    # Write output file to fileStore
    ids['uuid.tar.gz'] = job.fileStore.writeGlobalFile(out_tar)
    # If S3 bucket argument specified, upload to S3
    if input_args['s3_dir']:
        job.addChildJobFn(upload_output_to_s3, job_vars)

Response: Combine the contents of separate zipped outputs into one.


Instruction:
Implement a Python 3 function for
uploading the output of the job to S3 using boto.

Response:
def upload_output_to_s3(job, job_vars):
    """
    If s3_dir is specified in arguments, file will be uploaded to S3 using boto.
    WARNING: ~/.boto credentials are necessary for this to succeed!

    job_vars: tuple     Tuple of dictionaries: input_args and ids
    """
    import boto
    from boto.s3.key import Key

    input_args, ids = job_vars
    work_dir = job.fileStore.getLocalTempDir()
    uuid = input_args['uuid']
    # Parse s3_dir
    s3_dir = input_args['s3_dir']
    bucket_name = s3_dir.split('/')[0]
    bucket_dir = '/'.join(s3_dir.split('/')[1:])
    # I/O
    uuid_tar = return_input_paths(job, work_dir, ids, 'uuid.tar.gz')
    # Upload to S3 via boto
    conn = boto.connect_s3()
    bucket = conn.get_bucket(bucket_name)
    k = Key(bucket)
    k.key = os.path.join(bucket_dir, uuid + '.tar.gz')
    k.set_contents_from_filename(uuid_tar)


Instruction:
Implement a function in Python 3 to
upload bam to S3

Response:
def upload_bam_to_s3(job, job_vars):
    """
    Upload bam to S3. Requires S3AM and a ~/.boto config file.
    """
    input_args, ids = job_vars
    work_dir = job.fileStore.getLocalTempDir()
    uuid = input_args['uuid']
    # I/O
    job.fileStore.readGlobalFile(ids['alignments.bam'], os.path.join(work_dir, 'alignments.bam'))
    bam_path = os.path.join(work_dir, 'alignments.bam')
    sample_name = uuid + '.bam'
    # Parse s3_dir to get bucket and s3 path
    s3_dir = input_args['s3_dir']
    bucket_name = s3_dir.split('/')[0]
    bucket_dir = os.path.join('/'.join(s3_dir.split('/')[1:]), 'bam_files')
    # Upload to S3 via S3AM
    s3am_command = ['s3am',
                    'upload',
                    'file://{}'.format(bam_path),
                    os.path.join('s3://', bucket_name, bucket_dir, sample_name)]
    subprocess.check_call(s3am_command)


Instruction:
Can you generate the documentation for the following Python 3 function
def main():
    # Define Parser object and add to toil
    parser = build_parser()
    Job.Runner.addToilOptions(parser)
    args = parser.parse_args()
    # Store inputs from argparse
    inputs = {'config': args.config,
              'config_fastq': args.config_fastq,
              'input': args.input,
              'unc.bed': args.unc,
              'hg19.transcripts.fa': args.fasta,
              'composite_exons.bed': args.composite_exons,
              'normalize.pl': args.normalize,
              'output_dir': args.output_dir,
              'rsem_ref.zip': args.rsem_ref,
              'chromosomes.zip': args.chromosomes,
              'ebwt.zip': args.ebwt,
              'ssec': args.ssec,
              's3_dir': args.s3_dir,
              'sudo': args.sudo,
              'single_end_reads': args.single_end_reads,
              'upload_bam_to_s3': args.upload_bam_to_s3,
              'uuid': None,
              'sample.tar': None,
              'cpu_count': None}

    # Launch jobs
    Job.Runner.startToil(Job.wrapJobFn(download_shared_files, inputs), args)

Response: This is the main function for the UNC best practice RNA - Seq analysis. It is a Toil pipeline for the UNC best practice RNA - Seq analysis. It is a Toil pipeline for the UNC best practice RNA - Seq analysis.


Instruction:
Write a Python 3 script to
remove the given file from hdfs with the given IP address.

Response:
def remove_file(master_ip, filename, spark_on_toil):
    """
    Remove the given file from hdfs with master at the given IP address

    :type masterIP: MasterAddress
    """
    master_ip = master_ip.actual

    ssh_call = ['ssh', '-o', 'StrictHostKeyChecking=no', master_ip]

    if spark_on_toil:
        output = check_output(ssh_call + ['docker', 'ps'])
        container_id = next(line.split()[0] for line in output.splitlines() if 'apache-hadoop-master' in line)
        ssh_call += ['docker', 'exec', container_id]

    try:
        check_call(ssh_call + ['hdfs', 'dfs', '-rm', '-r', '/' + filename])
    except:
        pass


Instruction:
Make a summary of the following Python 3 code
def download_data(job, master_ip, inputs, known_snps, bam, hdfs_snps, hdfs_bam):

    log.info("Downloading known sites file %s to %s.", known_snps, hdfs_snps)
    call_conductor(job, master_ip, known_snps, hdfs_snps, memory=inputs.memory)

    log.info("Downloading input BAM %s to %s.", bam, hdfs_bam)
    call_conductor(job, master_ip, bam, hdfs_bam, memory=inputs.memory)

Response: Downloads input data files from S3.


Instruction:
Implement a function in Python 3 to
convert input sam file and known SNPs file into ADAM format.

Response:
def adam_convert(job, master_ip, inputs, in_file, in_snps, adam_file, adam_snps, spark_on_toil):
    """
    Convert input sam/bam file and known SNPs file into ADAM format
    """

    log.info("Converting input BAM to ADAM.")
    call_adam(job, master_ip,
              ["transform", in_file, adam_file],
              memory=inputs.memory,
              run_local=inputs.run_local,
              native_adam_path=inputs.native_adam_path)

    in_file_name = in_file.split("/")[-1]
    remove_file(master_ip, in_file_name, spark_on_toil)

    log.info("Converting known sites VCF to ADAM.")

    call_adam(job, master_ip,
              ["vcf2adam", "-only_variants", in_snps, adam_snps],
              memory=inputs.memory,
              run_local=inputs.run_local,
              native_adam_path=inputs.native_adam_path)

    in_snps_name = in_snps.split("/")[-1]
    remove_file(master_ip, in_snps_name, spark_on_toil)


Instruction:
Can you create a Python 3 function that
transforms an ADAT file into a single file.

Response:
def adam_transform(job, master_ip, inputs, in_file, snp_file, hdfs_dir, out_file, spark_on_toil):
    """
    Preprocess in_file with known SNPs snp_file:
        - mark duplicates
        - realign indels
        - recalibrate base quality scores
    """

    log.info("Marking duplicate reads.")
    call_adam(job, master_ip,
              ["transform",
               in_file,  hdfs_dir + "/mkdups.adam",
               "-aligned_read_predicate",
               "-limit_projection",
               "-mark_duplicate_reads"],
              memory=inputs.memory,
              run_local=inputs.run_local,
              native_adam_path=inputs.native_adam_path)

    #FIXME
    in_file_name = in_file.split("/")[-1]
    remove_file(master_ip, in_file_name + "*", spark_on_toil)

    log.info("Realigning INDELs.")
    call_adam(job, master_ip,
              ["transform",
               hdfs_dir + "/mkdups.adam",
               hdfs_dir + "/ri.adam",
               "-realign_indels"],
              memory=inputs.memory,
              run_local=inputs.run_local,
              native_adam_path=inputs.native_adam_path)

    remove_file(master_ip, hdfs_dir + "/mkdups.adam*", spark_on_toil)

    log.info("Recalibrating base quality scores.")
    call_adam(job, master_ip,
              ["transform",
               hdfs_dir + "/ri.adam",
               hdfs_dir + "/bqsr.adam",
               "-recalibrate_base_qualities",
               "-known_snps", snp_file],
              memory=inputs.memory,
              run_local=inputs.run_local,
              native_adam_path=inputs.native_adam_path)

    remove_file(master_ip, "ri.adam*", spark_on_toil)

    log.info("Sorting reads and saving a single BAM file.")
    call_adam(job, master_ip,
              ["transform",
               hdfs_dir + "/bqsr.adam",
               out_file,
               "-sort_reads", "-single"],
              memory=inputs.memory,
              run_local=inputs.run_local,
              native_adam_path=inputs.native_adam_path)

    remove_file(master_ip, "bqsr.adam*", spark_on_toil)

    return out_file


Instruction:
Implement a function in Python 3 to
upload file hdfsName from hdfs to s3

Response:
def upload_data(job, master_ip, inputs, hdfs_name, upload_name, spark_on_toil):
    """
    Upload file hdfsName from hdfs to s3
    """

    if mock_mode():
        truncate_file(master_ip, hdfs_name, spark_on_toil)

    log.info("Uploading output BAM %s to %s.", hdfs_name, upload_name)
    call_conductor(job, master_ip, hdfs_name, upload_name, memory=inputs.memory)
    remove_file(master_ip, hdfs_name, spark_on_toil)


Instruction:
How would you implement a function in Python 3 that
downloads and upload the data from the master and store it in the job store.

Response:
def download_run_and_upload(job, master_ip, inputs, spark_on_toil):
    """
    Monolithic job that calls data download, conversion, transform, upload.
    Previously, this was not monolithic; change came in due to #126/#134.
    """
    master_ip = MasterAddress(master_ip)

    bam_name = inputs.sample.split('://')[-1].split('/')[-1]
    sample_name = ".".join(os.path.splitext(bam_name)[:-1])

    hdfs_subdir = sample_name + "-dir"

    if inputs.run_local:
        inputs.local_dir = job.fileStore.getLocalTempDir()
        if inputs.native_adam_path is None:
            hdfs_dir = "/data/"
        else:
            hdfs_dir = inputs.local_dir
    else:
        inputs.local_dir = None
        hdfs_dir = "hdfs://{0}:{1}/{2}".format(master_ip, HDFS_MASTER_PORT, hdfs_subdir)

    try:
        hdfs_prefix = hdfs_dir + "/" + sample_name
        hdfs_bam = hdfs_dir + "/" + bam_name

        hdfs_snps = hdfs_dir + "/" + inputs.dbsnp.split('://')[-1].split('/')[-1]

        if not inputs.run_local:
            download_data(job, master_ip, inputs, inputs.dbsnp, inputs.sample, hdfs_snps, hdfs_bam)
        else:
            copy_files([inputs.sample, inputs.dbsnp], inputs.local_dir)

        adam_input = hdfs_prefix + ".adam"
        adam_snps = hdfs_dir + "/snps.var.adam"
        adam_convert(job, master_ip, inputs, hdfs_bam, hdfs_snps, adam_input, adam_snps, spark_on_toil)

        adam_output = hdfs_prefix + ".processed.bam"
        adam_transform(job, master_ip, inputs, adam_input, adam_snps, hdfs_dir, adam_output, spark_on_toil)

        out_file = inputs.output_dir + "/" + sample_name + inputs.suffix + ".bam"

        if not inputs.run_local:
            upload_data(job, master_ip, inputs, adam_output, out_file, spark_on_toil)
        else:
            local_adam_output = "%s/%s.processed.bam" % (inputs.local_dir, sample_name)
            move_files([local_adam_output], inputs.output_dir)

        remove_file(master_ip, hdfs_subdir, spark_on_toil)
    except:
        remove_file(master_ip, hdfs_subdir, spark_on_toil)
        raise


Instruction:
Make a summary of the following Python 3 code
def hard_filter_pipeline(job, uuid, vcf_id, config):
    job.fileStore.logToMaster('Running Hard Filter on {}'.format(uuid))

    # Get the total size of the genome reference
    genome_ref_size = config.genome_fasta.size + config.genome_fai.size + config.genome_dict.size

    # The SelectVariants disk requirement depends on the input VCF, the genome reference files,
    # and the output VCF. The output VCF is smaller than the input VCF. The disk requirement
    # is identical for SNPs and INDELs.
    select_variants_disk = PromisedRequirement(lambda vcf, ref_size: 2 * vcf.size + ref_size,
                                               vcf_id,
                                               genome_ref_size)
    select_snps = job.wrapJobFn(gatk_select_variants,
                                'SNP',
                                vcf_id,
                                config.genome_fasta,
                                config.genome_fai,
                                config.genome_dict,
                                memory=config.xmx,
                                disk=select_variants_disk)

    # The VariantFiltration disk requirement depends on the input VCF, the genome reference files,
    # and the output VCF. The filtered VCF is smaller than the input VCF.
    snp_filter_disk = PromisedRequirement(lambda vcf, ref_size: 2 * vcf.size + ref_size,
                                          select_snps.rv(),
                                          genome_ref_size)

    snp_filter = job.wrapJobFn(gatk_variant_filtration,
                               select_snps.rv(),
                               config.snp_filter_name,
                               config.snp_filter_expression,
                               config.genome_fasta,
                               config.genome_fai,
                               config.genome_dict,
                               memory=config.xmx,
                               disk=snp_filter_disk)

    select_indels = job.wrapJobFn(gatk_select_variants,
                                  'INDEL',
                                  vcf_id,
                                  config.genome_fasta,
                                  config.genome_fai,
                                  config.genome_dict,
                                  memory=config.xmx,
                                  disk=select_variants_disk)

    indel_filter_disk = PromisedRequirement(lambda vcf, ref_size: 2 * vcf.size + ref_size,
                                            select_indels.rv(),
                                            genome_ref_size)

    indel_filter = job.wrapJobFn(gatk_variant_filtration,
                                 select_indels.rv(),
                                 config.indel_filter_name,
                                 config.indel_filter_expression,
                                 config.genome_fasta,
                                 config.genome_fai,
                                 config.genome_dict,
                                 memory=config.xmx,
                                 disk=indel_filter_disk)

    # The CombineVariants disk requirement depends on the SNP and INDEL input VCFs and the
    # genome reference files. The combined VCF is approximately the same size as the input files.
    combine_vcfs_disk = PromisedRequirement(lambda vcf1, vcf2, ref_size:
                                            2 * (vcf1.size + vcf2.size) + ref_size,
                                            indel_filter.rv(),
                                            snp_filter.rv(),
                                            genome_ref_size)

    combine_vcfs = job.wrapJobFn(gatk_combine_variants,
                                 {'SNPs': snp_filter.rv(), 'INDELs': indel_filter.rv()},
                                 config.genome_fasta,
                                 config.genome_fai,
                                 config.genome_dict,
                                 merge_option='UNSORTED',  # Merges variants from a single sample
                                 memory=config.xmx,
                                 disk=combine_vcfs_disk)

    job.addChild(select_snps)
    job.addChild(select_indels)

    select_snps.addChild(snp_filter)
    snp_filter.addChild(combine_vcfs)

    select_indels.addChild(indel_filter)
    indel_filter.addChild(combine_vcfs)

    # Output the hard filtered VCF
    output_dir = os.path.join(config.output_dir, uuid)
    output_filename = '%s.hard_filter%s.vcf' % (uuid, config.suffix)
    output_vcf = job.wrapJobFn(output_file_job,
                               output_filename,
                               combine_vcfs.rv(),
                               output_dir,
                               s3_key_path=config.ssec,
                               disk=PromisedRequirement(lambda x: x.size, combine_vcfs.rv()))
    combine_vcfs.addChild(output_vcf)
    return combine_vcfs.rv()

Response: This function runs GATK Hard Filtering on a Genomic VCF file and uploads the results to the output directory.


Instruction:
Make a summary of the following Python 3 code
def download_and_transfer_sample(job, sample, inputs):

    """
    Downloads a sample from CGHub via GeneTorrent, then uses S3AM to transfer it to S3

    input_args: dict        Dictionary of input arguments
    analysis_id: str        An analysis ID for a sample in CGHub
    """
    analysis_id = sample[0]
    work_dir = job.fileStore.getLocalTempDir()
    folder_path = os.path.join(work_dir, os.path.basename(analysis_id))
    # Acquire genetorrent key and download sample
    shutil.copy(inputs['genetorrent_key'], os.path.join(work_dir, 'cghub.key'))
    parameters = ['-vv', '-c', 'cghub.key', '-d', analysis_id]
    docker_call(job=job, tool='quay.io/ucsc_cgl/genetorrent:3.8.7--9911761265b6f08bc3ef09f53af05f56848d805b',
                work_dir=work_dir, parameters=parameters)
    try:
        sample = glob.glob(os.path.join(folder_path, '*tar*'))[0]
    except KeyError as e:
        print 'No tarfile found inside of folder: '.format(e)
        raise
    # Upload sample to S3AM
    key_path = inputs['ssec']
    if sample.endswith('gz'):
        sample_name = analysis_id + '.tar.gz'
        shutil.move(sample, os.path.join(work_dir, sample_name))
    else:
        sample_name = analysis_id + '.tar'
        shutil.move(sample, os.path.join(work_dir, sample_name))
    # Parse s3_dir to get bucket and s3 path
    s3_dir = inputs['s3_dir']
    bucket_name = s3_dir.lstrip('/').split('/')[0]
    base_url = 'https://s3-us-west-2.amazonaws.com/'
    url = os.path.join(base_url, bucket_name, sample_name)
    # Generate keyfile for upload
    with open(os.path.join(work_dir, 'temp.key'), 'wb') as f_out:
        f_out.write(generate_unique_key(key_path, url))
    # Upload to S3 via S3AM
    s3am_command = ['s3am',
                    'upload',
                    '--sse-key-file', os.path.join(work_dir, 'temp.key'),
                    'file://{}'.format(os.path.join(work_dir, sample_name)),
                    's3://' + bucket_name + '/']
    subprocess.check_call(s3am_command)

Response: Downloads a sample from CGHub then uses S3AM to transfer it to CGHub


Instruction:
Write a Python 3 script for
validating a hexidecimal IPv6 ip address.

Response:
def validate_ip(s):
    """Validate a hexidecimal IPv6 ip address.

    >>> validate_ip('::')
    True
    >>> validate_ip('::1')
    True
    >>> validate_ip('2001:db8:85a3::8a2e:370:7334')
    True
    >>> validate_ip('2001:db8:85a3:0:0:8a2e:370:7334')
    True
    >>> validate_ip('2001:0db8:85a3:0000:0000:8a2e:0370:7334')
    True
    >>> validate_ip('2001:db8::1:0:0:1')
    True
    >>> validate_ip('ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff')
    True
    >>> validate_ip('::ffff:192.0.2.128')
    True
    >>> validate_ip('::ff::ff')
    False
    >>> validate_ip('::fffff')
    False
    >>> validate_ip('::ffff:192.0.2.300')
    False
    >>> validate_ip(None) #doctest: +IGNORE_EXCEPTION_DETAIL
    Traceback (most recent call last):
        ...
    TypeError: expected string or buffer
    >>> validate_ip('1080:0:0:0:8:800:200c:417a')
    True

    :param s: String to validate as a hexidecimal IPv6 ip address.
    :type s: str
    :returns: ``True`` if a valid hexidecimal IPv6 ip address,
              ``False`` otherwise.
    :raises: TypeError
    """
    if _HEX_RE.match(s):
        return len(s.split('::')) <= 2
    if _DOTTED_QUAD_RE.match(s):
        halves = s.split('::')
        if len(halves) > 2:
            return False
        hextets = s.split(':')
        quads = hextets[-1].split('.')
        for q in quads:
            if int(q) > 255:
                return False
        return True
    return False


Instruction:
How would you explain what the following Python 3 function does
def ip2long(ip):
    if not validate_ip(ip):
        return None

    if '.' in ip:
        # convert IPv4 suffix to hex
        chunks = ip.split(':')
        v4_int = ipv4.ip2long(chunks.pop())
        if v4_int is None:
            return None
        chunks.append('%x' % ((v4_int >> 16) & 0xffff))
        chunks.append('%x' % (v4_int & 0xffff))
        ip = ':'.join(chunks)

    halves = ip.split('::')
    hextets = halves[0].split(':')
    if len(halves) == 2:
        h2 = halves[1].split(':')
        for z in range(8 - (len(hextets) + len(h2))):
            hextets.append('0')
        for h in h2:
            hextets.append(h)
    # end if

    lngip = 0
    for h in hextets:
        if '' == h:
            h = '0'
        lngip = (lngip << 16) | int(h, 16)
    return lngip

Response: Convert a hexidecimal IPv6 address to a network byte order 128 - bit integer.


Instruction:
Write a Python 3 function that can
convert a network byte order 128 - bit integer to a canonical IPv6 address.

Response:
def long2ip(l, rfc1924=False):
    """Convert a network byte order 128-bit integer to a canonical IPv6
    address.

    >>> long2ip(2130706433)
    '::7f00:1'
    >>> long2ip(42540766411282592856904266426630537217)
    '2001:db8::1:0:0:1'
    >>> long2ip(MIN_IP)
    '::'
    >>> long2ip(MAX_IP)
    'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff'
    >>> long2ip(None) #doctest: +IGNORE_EXCEPTION_DETAIL
    Traceback (most recent call last):
        ...
    TypeError: unsupported operand type(s) for >>: 'NoneType' and 'int'
    >>> long2ip(-1) #doctest: +IGNORE_EXCEPTION_DETAIL
    Traceback (most recent call last):
        ...
    TypeError: expected int between 0 and <really big int> inclusive
    >>> long2ip(MAX_IP + 1) #doctest: +IGNORE_EXCEPTION_DETAIL
    Traceback (most recent call last):
        ...
    TypeError: expected int between 0 and <really big int> inclusive
    >>> long2ip(ip2long('1080::8:800:200C:417A'), rfc1924=True)
    '4)+k&C#VzJ4br>0wv%Yp'
    >>> long2ip(ip2long('::'), rfc1924=True)
    '00000000000000000000'

    :param l: Network byte order 128-bit integer.
    :type l: int
    :param rfc1924: Encode in RFC 1924 notation (base 85)
    :type rfc1924: bool
    :returns: Canonical IPv6 address (eg. '::1').
    :raises: TypeError
    """
    if MAX_IP < l or l < MIN_IP:
        raise TypeError(
            "expected int between %d and %d inclusive" % (MIN_IP, MAX_IP))

    if rfc1924:
        return long2rfc1924(l)

    # format as one big hex value
    hex_str = '%032x' % l
    # split into double octet chunks without padding zeros
    hextets = ['%x' % int(hex_str[x:x + 4], 16) for x in range(0, 32, 4)]

    # find and remove left most longest run of zeros
    dc_start, dc_len = (-1, 0)
    run_start, run_len = (-1, 0)
    for idx, hextet in enumerate(hextets):
        if '0' == hextet:
            run_len += 1
            if -1 == run_start:
                run_start = idx
            if run_len > dc_len:
                dc_len, dc_start = (run_len, run_start)
        else:
            run_len, run_start = (0, -1)
    # end for
    if dc_len > 1:
        dc_end = dc_start + dc_len
        if dc_end == len(hextets):
            hextets += ['']
        hextets[dc_start:dc_end] = ['']
        if dc_start == 0:
            hextets = [''] + hextets
    # end if

    return ':'.join(hextets)


Instruction:
Given the following Python 3 function, write the documentation
def long2rfc1924(l):
    if MAX_IP < l or l < MIN_IP:
        raise TypeError(
            "expected int between %d and %d inclusive" % (MIN_IP, MAX_IP))
    o = []
    r = l
    while r > 85:
        o.append(_RFC1924_ALPHABET[r % 85])
        r = r // 85
    o.append(_RFC1924_ALPHABET[r])
    return ''.join(reversed(o)).zfill(20)

Response: Convert a network byte order 128 - bit integer to an RFC 1924 IPv6 address.


Instruction:
Here you have a function in Python 3, explain what it does
def rfc19242long(s):
    global _RFC1924_REV
    if not _RFC1924_RE.match(s):
        return None
    if _RFC1924_REV is None:
        _RFC1924_REV = {v: k for k, v in enumerate(_RFC1924_ALPHABET)}
    x = 0
    for c in s:
        x = x * 85 + _RFC1924_REV[c]
    if x > MAX_IP:
        return None
    return x

Response: Convert an RFC 1924 IPv6 address to a network byte order 128 - bit integer.


Instruction:
Write a Python 3 script for
validating a CIDR notation ip address.

Response:
def validate_cidr(s):
    """Validate a CIDR notation ip address.

    The string is considered a valid CIDR address if it consists of a valid
    IPv6 address in hextet format followed by a forward slash (/) and a bit
    mask length (0-128).

    >>> validate_cidr('::/128')
    True
    >>> validate_cidr('::/0')
    True
    >>> validate_cidr('fc00::/7')
    True
    >>> validate_cidr('::ffff:0:0/96')
    True
    >>> validate_cidr('::')
    False
    >>> validate_cidr('::/129')
    False
    >>> validate_cidr(None) #doctest: +IGNORE_EXCEPTION_DETAIL
    Traceback (most recent call last):
        ...
    TypeError: expected string or buffer

    :param s: String to validate as a CIDR notation ip address.
    :type s: str
    :returns: ``True`` if a valid CIDR address, ``False`` otherwise.
    :raises: TypeError
    """
    if _CIDR_RE.match(s):
        ip, mask = s.split('/')
        if validate_ip(ip):
            if int(mask) > 128:
                return False
        else:
            return False
        return True
    return False


Instruction:
Can you generate a brief explanation for the following Python 3 code
def cidr2block(cidr):
    if not validate_cidr(cidr):
        return None

    ip, prefix = cidr.split('/')
    prefix = int(prefix)
    ip = ip2long(ip)

    # keep left most prefix bits of ip
    shift = 128 - prefix
    block_start = ip >> shift << shift

    # expand right most 128 - prefix bits to 1
    mask = (1 << shift) - 1
    block_end = block_start | mask
    return (long2ip(block_start), long2ip(block_end))

Response:
Convert a CIDR notation ip address into a tuple containing the network
    block start and end addresses.


Instruction:
Can you create a Python 3 function that
parses the input samples and adds them to the job.

Response:
def parse_input_samples(job, inputs):
    """
    Parses config file to pull sample information.
    Stores samples as tuples of (uuid, URL)

    :param JobFunctionWrappingJob job: passed by Toil automatically
    :param Namespace inputs: Stores input arguments (see main)
    """
    job.fileStore.logToMaster('Parsing input samples and batching jobs')
    samples = []
    if inputs.config:
        with open(inputs.config, 'r') as f:
            for line in f.readlines():
                if not line.isspace():
                    sample = line.strip().split(',')
                    assert len(sample) == 2, 'Error: Config file is inappropriately formatted.'
                    samples.append(sample)
    job.addChildJobFn(map_job, download_sample, samples, inputs)


Instruction:
Can you generate the documentation for the following Python 3 function
def download_sample(job, sample, inputs):
    uuid, url = sample
    job.fileStore.logToMaster('Downloading sample: {}'.format(uuid))
    # Download sample
    tar_id = job.addChildJobFn(download_url_job, url, s3_key_path=inputs.ssec, disk='30G').rv()
    # Create copy of inputs for each sample
    sample_inputs = argparse.Namespace(**vars(inputs))
    sample_inputs.uuid = uuid
    sample_inputs.cores = multiprocessing.cpu_count()
    # Call children and follow-on jobs
    job.addFollowOnJobFn(process_sample, sample_inputs, tar_id, cores=2, disk='60G')

Response: Download the sample from Toil and return the tar_id


Instruction:
Can you implement a function in Python 3 that
converts sample.tar(.gz) into two fastq files. Due to edge conditions... BEWARE: HERE BE DRAGONS :param JobFunctionWrappingJob job: passed by Toil automatically :param Namespace inputs: Stores input arguments (see main) :param str tar_id: FileStore ID of sample tar

Response:
def process_sample(job, inputs, tar_id):
    """
    Converts sample.tar(.gz) into two fastq files.
    Due to edge conditions... BEWARE: HERE BE DRAGONS

    :param JobFunctionWrappingJob job: passed by Toil automatically
    :param Namespace inputs: Stores input arguments (see main)
    :param str tar_id: FileStore ID of sample tar
    """
    job.fileStore.logToMaster('Processing sample into read pairs: {}'.format(inputs.uuid))
    work_dir = job.fileStore.getLocalTempDir()
    # I/O
    tar_path = job.fileStore.readGlobalFile(tar_id, os.path.join(work_dir, 'sample.tar'))
    # Untar File and concat
    subprocess.check_call(['tar', '-xvf', tar_path, '-C', work_dir])
    os.remove(os.path.join(work_dir, 'sample.tar'))
    # Grab files from tarball
    fastqs = []
    for root, subdir, files in os.walk(work_dir):
        fastqs.extend([os.path.join(root, x) for x in files])
    # Check for read 1 and read 2 files
    r1 = sorted([x for x in fastqs if 'R1' in x])
    r2 = sorted([x for x in fastqs if 'R2' in x])
    if not r1 or not r2:
        # Check if using a different standard
        r1 = sorted([x for x in fastqs if '_1' in x])
        r2 = sorted([x for x in fastqs if '_2' in x])
    # Prune file name matches from each list
    if len(r1) > len(r2):
        r1 = [x for x in r1 if x not in r2]
    elif len(r2) > len(r1):
        r2 = [x for x in r2 if x not in r1]
    # Flag if data is single-ended
    assert r1 and r2, 'This pipeline does not support single-ended data. R1: {}\nR2:{}'.format(r1, r2)
    command = 'zcat' if r1[0].endswith('gz') and r2[0].endswith('gz') else 'cat'
    with open(os.path.join(work_dir, 'R1.fastq'), 'w') as f1:
        p1 = subprocess.Popen([command] + r1, stdout=f1)
    with open(os.path.join(work_dir, 'R2.fastq'), 'w') as f2:
        p2 = subprocess.Popen([command] + r2, stdout=f2)
    p1.wait()
    p2.wait()
    # Write to fileStore
    r1_id = job.fileStore.writeGlobalFile(os.path.join(work_dir, 'R1.fastq'))
    r2_id = job.fileStore.writeGlobalFile(os.path.join(work_dir, 'R2.fastq'))
    job.fileStore.deleteGlobalFile(tar_id)
    # Start cutadapt step
    job.addChildJobFn(cutadapt, inputs, r1_id, r2_id, disk='60G').rv()


Instruction:
Can you write a function in Python 3 where it
filters out adapters that may be left in the RNA-seq files :param JobFunctionWrappingJob job: passed by Toil automatically :param Namespace inputs: Stores input arguments (see main) :param str r1_id: FileStore ID of read 1 fastq :param str r2_id: FileStore ID of read 2 fastq

Response:
def cutadapt(job, inputs, r1_id, r2_id):
    """
    Filters out adapters that may be left in the RNA-seq files

    :param JobFunctionWrappingJob job: passed by Toil automatically
    :param Namespace inputs: Stores input arguments (see main)
    :param str r1_id: FileStore ID of read 1 fastq
    :param str r2_id: FileStore ID of read 2 fastq
    """
    job.fileStore.logToMaster('Running CutAdapt: {}'.format(inputs.uuid))
    work_dir = job.fileStore.getLocalTempDir()
    inputs.improper_pair = None
    # Retrieve files
    job.fileStore.readGlobalFile(r1_id, os.path.join(work_dir, 'R1.fastq'))
    job.fileStore.readGlobalFile(r2_id, os.path.join(work_dir, 'R2.fastq'))
    # Cutadapt parameters
    parameters = ['-a', inputs.fwd_3pr_adapter,
                  '-m', '35',
                  '-A', inputs.rev_3pr_adapter,
                  '-o', '/data/R1_cutadapt.fastq',
                  '-p', '/data/R2_cutadapt.fastq',
                  '/data/R1.fastq', '/data/R2.fastq']
    # Call: CutAdapt
    base_docker_call = 'docker run --log-driver=none --rm -v {}:/data'.format(work_dir).split()
    if inputs.sudo:
        base_docker_call = ['sudo'] + base_docker_call
    tool = 'quay.io/ucsc_cgl/cutadapt:1.9--6bd44edd2b8f8f17e25c5a268fedaab65fa851d2'
    p = subprocess.Popen(base_docker_call + [tool] + parameters, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
    stdout, stderr = p.communicate()
    if p.returncode != 0:
        if 'improperly paired' in stderr:
            inputs.improper_pair = True
            shutil.move(os.path.join(work_dir, 'R1.fastq'), os.path.join(work_dir, 'R1_cutadapt.fastq'))
            shutil.move(os.path.join(work_dir, 'R2.fastq'), os.path.join(work_dir, 'R2_cutadapt.fastq'))
    # Write to fileStore
    if inputs.improper_pair:
        r1_cutadapt = r1_id
        r2_cutadapt = r2_id
    else:
        r1_cutadapt = job.fileStore.writeGlobalFile(os.path.join(work_dir, 'R1_cutadapt.fastq'))
        r2_cutadapt = job.fileStore.writeGlobalFile(os.path.join(work_dir, 'R2_cutadapt.fastq'))
        job.fileStore.deleteGlobalFile(r1_id)
        job.fileStore.deleteGlobalFile(r2_id)
    # start STAR
    cores = min(inputs.cores, 16)
    job.addChildJobFn(star, inputs, r1_cutadapt, r2_cutadapt, cores=cores, disk='100G', memory='40G').rv()


Instruction:
How would you explain what the following Python 3 function does
def star(job, inputs, r1_cutadapt, r2_cutadapt):
    job.fileStore.logToMaster('Aligning with STAR: {}'.format(inputs.uuid))
    work_dir = job.fileStore.getLocalTempDir()
    cores = min(inputs.cores, 16)
    # Retrieve files
    job.fileStore.readGlobalFile(r1_cutadapt, os.path.join(work_dir, 'R1_cutadapt.fastq'))
    job.fileStore.readGlobalFile(r2_cutadapt, os.path.join(work_dir, 'R2_cutadapt.fastq'))
    # Get starIndex
    download_url(job=job, url=inputs.star_index, work_dir=work_dir, name='starIndex.tar.gz')
    subprocess.check_call(['tar', '-xvf', os.path.join(work_dir, 'starIndex.tar.gz'), '-C', work_dir])
    # Parameters
    parameters = ['--runThreadN', str(cores),
                  '--genomeDir', '/data/starIndex',
                  '--outFileNamePrefix', 'rna',
                  '--outSAMtype', 'BAM', 'SortedByCoordinate',
                  '--outSAMunmapped', 'Within',
                  '--quantMode', 'TranscriptomeSAM',
                  '--outSAMattributes', 'NH', 'HI', 'AS', 'NM', 'MD',
                  '--outFilterType', 'BySJout',
                  '--outFilterMultimapNmax', '20',
                  '--outFilterMismatchNmax', '999',
                  '--outFilterMismatchNoverReadLmax', '0.04',
                  '--alignIntronMin', '20',
                  '--alignIntronMax', '1000000',
                  '--alignMatesGapMax', '1000000',
                  '--alignSJoverhangMin', '8',
                  '--alignSJDBoverhangMin', '1',
                  '--sjdbScore', '1',
                  '--readFilesIn', '/data/R1_cutadapt.fastq', '/data/R2_cutadapt.fastq']
    # Call: STAR Map
    docker_call(job=job, tool='quay.io/ucsc_cgl/star:2.4.2a--bcbd5122b69ff6ac4ef61958e47bde94001cfe80',
                work_dir=work_dir, parameters=parameters)
    # Call Samtools Index
    index_command = ['index', '/data/rnaAligned.sortedByCoord.out.bam']
    docker_call(job=job, work_dir=work_dir, parameters=index_command,
                tool='quay.io/ucsc_cgl/samtools:1.3--256539928ea162949d8a65ca5c79a72ef557ce7c')
    # fileStore
    bam_id = job.fileStore.writeGlobalFile(os.path.join(work_dir, 'rnaAligned.sortedByCoord.out.bam'))
    bai_id = job.fileStore.writeGlobalFile(os.path.join(work_dir, 'rnaAligned.sortedByCoord.out.bam.bai'))
    job.fileStore.deleteGlobalFile(r1_cutadapt)
    job.fileStore.deleteGlobalFile(r2_cutadapt)
    # Launch children and follow-on
    vcqc_id = job.addChildJobFn(variant_calling_and_qc, inputs, bam_id, bai_id, cores=2, disk='30G').rv()
    spladder_id = job.addChildJobFn(spladder, inputs, bam_id, bai_id, disk='30G').rv()
    job.addFollowOnJobFn(consolidate_output_tarballs, inputs, vcqc_id, spladder_id, disk='30G')

Response: This function aligns fastqs to BAM via STAR


Instruction:
Create a Python 3 function to
perform variant calling with samtools nad QC with CheckBias

Response:
def variant_calling_and_qc(job, inputs, bam_id, bai_id):
    """
    Perform variant calling with samtools nad QC with CheckBias

    :param JobFunctionWrappingJob job: passed by Toil automatically
    :param Namespace inputs: Stores input arguments (see main)
    :param str bam_id: FileStore ID of bam
    :param str bai_id: FileStore ID of bam index file
    :return: FileStore ID of qc tarball
    :rtype: str
    """
    job.fileStore.logToMaster('Variant calling and QC: {}'.format(inputs.uuid))
    work_dir = job.fileStore.getLocalTempDir()
    # Pull in alignment.bam from fileStore
    job.fileStore.readGlobalFile(bam_id, os.path.join(work_dir, 'alignment.bam'))
    job.fileStore.readGlobalFile(bai_id, os.path.join(work_dir, 'alignment.bam.bai'))
    # Download input files
    input_info = [(inputs.genome, 'genome.fa'), (inputs.positions, 'positions.tsv'),
                  (inputs.genome_index, 'genome.fa.fai'), (inputs.gtf, 'annotation.gtf'),
                  (inputs.gtf_m53, 'annotation.m53')]
    for url, fname in input_info:
        download_url(job=job, url=url, work_dir=work_dir, name=fname)

    # Part 1: Variant Calling
    variant_command = ['mpileup',
                       '-f', 'genome.fa',
                       '-l', 'positions.tsv',
                       '-v', 'alignment.bam',
                       '-t', 'DP,SP,INFO/AD,INFO/ADF,INFO/ADR,INFO/DPR,SP',
                       '-o', '/data/output.vcf.gz']
    docker_call(job=job, work_dir=work_dir, parameters=variant_command,
                tool='quay.io/ucsc_cgl/samtools:1.3--256539928ea162949d8a65ca5c79a72ef557ce7c')

    # Part 2: QC
    qc_command = ['-o', 'qc',
                  '-n', 'alignment.bam',
                  '-a', 'annotation.gtf',
                  '-m', 'annotation.m53']
    docker_call(job=job, work_dir=work_dir, parameters=qc_command,
                tool='jvivian/checkbias:612f129--b08a1fb6526a620bbb0304b08356f2ae7c3c0ec3')
    # Write output to fileStore and return ids
    output_tsv = glob(os.path.join(work_dir, '*counts.tsv*'))[0]
    output_vcf = os.path.join(work_dir, 'output.vcf.gz')
    tarball_files('vcqc.tar.gz', file_paths=[output_tsv, output_vcf], output_dir=work_dir)
    return job.fileStore.writeGlobalFile(os.path.join(work_dir, 'vcqc.tar.gz'))


Instruction:
Can you generate a brief explanation for the following Python 3 code
def spladder(job, inputs, bam_id, bai_id):
    job.fileStore.logToMaster('SplAdder: {}'.format(inputs.uuid))
    work_dir = job.fileStore.getLocalTempDir()
    # Pull in alignment.bam from fileStore
    job.fileStore.readGlobalFile(bam_id, os.path.join(work_dir, 'alignment.bam'))
    job.fileStore.readGlobalFile(bai_id, os.path.join(work_dir, 'alignment.bam.bai'))
    # Download input file
    download_url(job=job, url=inputs.gtf, work_dir=work_dir, name='annotation.gtf')
    download_url(job=job, url=inputs.gtf_pickle, work_dir=work_dir, name='annotation.gtf.pickle')
    # Call Spladder
    command = ['--insert_ir=y',
               '--insert_es=y',
               '--insert_ni=y',
               '--remove_se=n',
               '--validate_sg=n',
               '-b', 'alignment.bam',
               '-o ', '/data',
               '-a', 'annotation.gtf',
               '-v', 'y',
               '-c', '3',
               '-M', 'single',
               '-T', 'n',
               '-n', '50',
               '-P', 'y',
               '-p', 'n',
               '--sparse_bam', 'y']
    docker_call(job=job, work_dir=work_dir, parameters=command, sudo=inputs.sudo, tool='jvivian/spladder:1.0')
    # Write output to fileStore and return ids
    output_pickle = os.path.join(work_dir, ' ', 'spladder', 'genes_graph_conf3.alignment.pickle')
    if not os.path.exists(output_pickle):
        matches = []
        for root, dirnames, filenames in os.walk(work_dir):
            for filename in fnmatch.filter(filenames, '*genes_graph*'):
                matches.append(os.path.join(root, filename))
        if matches:
            output_pickle = matches[0]
        else:
            raise RuntimeError("Couldn't find genes file!")
    output_filt = os.path.join(work_dir, 'alignment.filt.hdf5')
    output = os.path.join(work_dir, 'alignment.hdf5')
    print os.listdir(work_dir)
    tarball_files('spladder.tar.gz', file_paths=[output_pickle, output_filt, output], output_dir=work_dir)
    return job.fileStore.writeGlobalFile(os.path.join(work_dir, 'spladder.tar.gz'))

Response: Run SplAdder to detect alternative splicing events


Instruction:
Implement a function in Python 3 to
combine the contents of separate tarballs into one.

Response:
def consolidate_output_tarballs(job, inputs, vcqc_id, spladder_id):
    """
    Combine the contents of separate tarballs into one.

    :param JobFunctionWrappingJob job: passed by Toil automatically
    :param Namespace inputs: Stores input arguments (see main)
    :param str vcqc_id: FileStore ID of variant calling and QC tarball
    :param str spladder_id: FileStore ID of spladder tarball
    """
    job.fileStore.logToMaster('Consolidating files and uploading: {}'.format(inputs.uuid))
    work_dir = job.fileStore.getLocalTempDir()
    # Retrieve IDs
    uuid = inputs.uuid
    # Unpack IDs
    # Retrieve output file paths to consolidate
    vcqc_tar = job.fileStore.readGlobalFile(vcqc_id, os.path.join(work_dir, 'vcqc.tar.gz'))
    spladder_tar = job.fileStore.readGlobalFile(spladder_id, os.path.join(work_dir, 'spladder.tar.gz'))
    # I/O
    fname = uuid + '.tar.gz' if not inputs.improper_pair else 'IMPROPER_PAIR' + uuid + '.tar.gz'
    out_tar = os.path.join(work_dir, fname)
    # Consolidate separate tarballs into one
    with tarfile.open(os.path.join(work_dir, out_tar), 'w:gz') as f_out:
        for tar in [vcqc_tar, spladder_tar]:
            with tarfile.open(tar, 'r') as f_in:
                for tarinfo in f_in:
                    with closing(f_in.extractfile(tarinfo)) as f_in_file:
                        if tar == vcqc_tar:
                            tarinfo.name = os.path.join(uuid, 'variants_and_qc', os.path.basename(tarinfo.name))
                        else:
                            tarinfo.name = os.path.join(uuid, 'spladder', os.path.basename(tarinfo.name))
                        f_out.addfile(tarinfo, fileobj=f_in_file)
    # Move to output directory
    if inputs.output_dir:
        mkdir_p(inputs.output_dir)
        shutil.copy(out_tar, os.path.join(inputs.output_dir, os.path.basename(out_tar)))
    # Upload to S3
    if inputs.output_s3_dir:
        out_id = job.fileStore.writeGlobalFile(out_tar)
        job.addChildJobFn(s3am_upload_job, file_id=out_id, s3_dir=inputs.output_s3_dir,
                          file_name=fname, key_path=inputs.ssec, cores=inputs.cores)


Instruction:
Can you tell what is the following Python 3 function doing
def main():
    # Define Parser object and add to toil
    url_prefix = 'https://s3-us-west-2.amazonaws.com/cgl-pipeline-inputs/'
    parser = argparse.ArgumentParser(description=main.__doc__, formatter_class=argparse.RawTextHelpFormatter)
    parser.add_argument('--config', required=True,
                        help='Path to configuration file for samples, one per line. UUID,URL_to_bamfile. '
                             'The URL may be a standard "http://", a "file://<abs_path>", or "s3://<bucket>/<key>"')
    parser.add_argument('--gtf', help='URL to annotation GTF file',
                        default=url_prefix + 'rnaseq_cgl/gencode.v23.annotation.gtf')
    parser.add_argument('--gtf-pickle', help='Pickled GTF file',
                        default=url_prefix + 'spladder/gencode.v23.annotation.gtf.pickle')
    parser.add_argument('--gtf-m53', help='M53 preprocessing annotation table',
                        default=url_prefix + 'spladder/gencode.v23.annotation.gtf.m53')
    parser.add_argument('--positions', help='URL to SNP positions over genes file (TSV)',
                        default=url_prefix + 'spladder/positions_fixed.tsv')
    parser.add_argument('--genome', help='URL to Genome fasta',
                        default=url_prefix + 'rnaseq_cgl/hg38_no_alt.fa')
    parser.add_argument('--genome-index', help='Index file (fai) of genome',
                        default=url_prefix + 'spladder/hg38_no_alt.fa.fai')
    parser.add_argument('--ssec', default=None, help='Path to master key used for downloading encrypted files.')
    parser.add_argument('--output-s3-dir', default=None, help='S3 Directory of the form: s3://bucket/directory')
    parser.add_argument('--output-dir', default=None, help='full path where final results will be output')
    parser.add_argument('--sudo', action='store_true', default=False,
                        help='Set flag if sudo is required to run Docker.')
    parser.add_argument('--star-index', help='URL to download STAR Index built from HG38/gencodev23 annotation.',
                        default=url_prefix + 'rnaseq_cgl/starIndex_hg38_no_alt.tar.gz')
    parser.add_argument('--fwd-3pr-adapter', help="Sequence for the FWD 3' Read Adapter.", default='AGATCGGAAGAG')
    parser.add_argument('--rev-3pr-adapter', help="Sequence for the REV 3' Read Adapter.", default='AGATCGGAAGAG')
    Job.Runner.addToilOptions(parser)
    args = parser.parse_args()
    # Sanity Checks
    if args.config:
        assert os.path.isfile(args.config), 'Config not found at: {}'.format(args.config)
    if args.ssec:
        assert os.path.isfile(args.ssec), 'Encryption key not found at: {}'.format(args.config)
    if args.output_s3_dir:
        assert args.output_s3_dir.startswith('s3://'), 'Wrong format for output s3 directory'
    # Program checks
    for program in ['curl', 'docker']:
        assert which(program), 'Program "{}" must be installed on every node.'.format(program)

    Job.Runner.startToil(Job.wrapJobFn(parse_input_samples, args), args)

Response: This function is the main function of the toil pipeline. It will aligns reads and performs alternative splicing analysis.


Instruction:
Given the following Python 3 function, write the documentation
def validate_ip(s):
    if _DOTTED_QUAD_RE.match(s):
        quads = s.split('.')
        for q in quads:
            if int(q) > 255:
                return False
        return True
    return False

Response: Validate a dotted - quad ip address.


Instruction:
Explain what the following Python 3 code does
def validate_netmask(s):
    if validate_ip(s):
        # Convert to binary string, strip '0b' prefix, 0 pad to 32 bits
        mask = bin(ip2network(s))[2:].zfill(32)
        # all left most bits must be 1, all right most must be 0
        seen0 = False
        for c in mask:
            if '1' == c:
                if seen0:
                    return False
            else:
                seen0 = True
        return True
    else:
        return False

Response: Validate that a dotted - quad ip address is a valid netmask.


Instruction:
Explain what the following Python 3 code does
def validate_subnet(s):
    if isinstance(s, basestring):
        if '/' in s:
            start, mask = s.split('/', 2)
            return validate_ip(start) and validate_netmask(mask)
        else:
            return False
    raise TypeError("expected string or unicode")

Response: Validate a dotted - quad ip address including a netmask.


Instruction:
Given the following Python 3 function, write the documentation
def ip2long(ip):
    if not validate_ip(ip):
        return None
    quads = ip.split('.')
    if len(quads) == 1:
        # only a network quad
        quads = quads + [0, 0, 0]
    elif len(quads) < 4:
        # partial form, last supplied quad is host address, rest is network
        host = quads[-1:]
        quads = quads[:-1] + [0, ] * (4 - len(quads)) + host

    lngip = 0
    for q in quads:
        lngip = (lngip << 8) | int(q)
    return lngip

Response: Convert a dotted - quad ip address to a network byte order 32 - bit integer.


Instruction:
Create a Python 3 function to
convert a dotted - quad ip address to base network number.

Response:
def ip2network(ip):
    """Convert a dotted-quad ip to base network number.

    This differs from :func:`ip2long` in that partial addresses as treated as
    all network instead of network plus host (eg. '127.1' expands to
    '127.1.0.0')

    :param ip: dotted-quad ip address (eg. ‘127.0.0.1’).
    :type ip: str
    :returns: Network byte order 32-bit integer or `None` if ip is invalid.
    """
    if not validate_ip(ip):
        return None
    quads = ip.split('.')
    netw = 0
    for i in range(4):
        netw = (netw << 8) | int(len(quads) > i and quads[i] or 0)
    return netw


Instruction:
Write a Python 3 function that can
convert a network byte order 32 - bit integer to a dotted quad ip.

Response:
def long2ip(l):
    """Convert a network byte order 32-bit integer to a dotted quad ip
    address.

    >>> long2ip(2130706433)
    '127.0.0.1'
    >>> long2ip(MIN_IP)
    '0.0.0.0'
    >>> long2ip(MAX_IP)
    '255.255.255.255'
    >>> long2ip(None) #doctest: +IGNORE_EXCEPTION_DETAIL
    Traceback (most recent call last):
        ...
    TypeError: unsupported operand type(s) for >>: 'NoneType' and 'int'
    >>> long2ip(-1) #doctest: +IGNORE_EXCEPTION_DETAIL
    Traceback (most recent call last):
        ...
    TypeError: expected int between 0 and 4294967295 inclusive
    >>> long2ip(374297346592387463875) #doctest: +IGNORE_EXCEPTION_DETAIL
    Traceback (most recent call last):
        ...
    TypeError: expected int between 0 and 4294967295 inclusive
    >>> long2ip(MAX_IP + 1) #doctest: +IGNORE_EXCEPTION_DETAIL
    Traceback (most recent call last):
        ...
    TypeError: expected int between 0 and 4294967295 inclusive

    :param l: Network byte order 32-bit integer.
    :type l: int
    :returns: Dotted-quad ip address (eg. '127.0.0.1').
    :raises: TypeError
    """
    if MAX_IP < l or l < MIN_IP:
        raise TypeError(
            "expected int between %d and %d inclusive" % (MIN_IP, MAX_IP))
    return '%d.%d.%d.%d' % (
        l >> 24 & 255, l >> 16 & 255, l >> 8 & 255, l & 255)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def cidr2block(cidr):
    if not validate_cidr(cidr):
        return None

    ip, prefix = cidr.split('/')
    prefix = int(prefix)

    # convert dotted-quad ip to base network number
    network = ip2network(ip)

    return _block_from_ip_and_prefix(network, prefix)

Response: Convert a CIDR notation ip address into a tuple containing the start and end addresses.


Instruction:
Make a summary of the following Python 3 code
def subnet2block(subnet):
    if not validate_subnet(subnet):
        return None

    ip, netmask = subnet.split('/')
    prefix = netmask2prefix(netmask)

    # convert dotted-quad ip to base network number
    network = ip2network(ip)

    return _block_from_ip_and_prefix(network, prefix)

Response: Convert a dotted - quad ip address including a netmask into a tuple of start and end addresses.