Instruction:
Create a Python 3 function to
run all the jobs in parallel

Response:
def run(self, ipyclient):
        """
        parallelize calls to worker function.
        """

        ## connect to parallel client
        lbview = ipyclient.load_balanced_view()

        ## iterate over tests
        asyncs = []
        for test in xrange(self.ntests):

            ## submit jobs to run
            async = lbview.apply(worker, self)
            asyncs.append(async)

        ## wait for jobs to finish
        ipyclient.wait()

        ## check for errors
        for async in asyncs:
            if not async.successful():
                raise Exception("Error: {}".format(async.result()))

        ## return results as df
        results = [i.result() for i in asyncs]
        self.results_table = pd.DataFrame(results)


Instruction:
Given the following Python 3 function, write the documentation
def plot(self):
        if self.results_table == None:
            return "no results found"
        else:
            bb = self.results_table.sort_values(
                by=["ABCD", "ACBD"],
                ascending=[False, True],
                )

            ## make a barplot
            import toyplot
            c = toyplot.Canvas(width=600, height=200)
            a = c.cartesian()
            m = a.bars(bb)
            return c, a, m

Response: return a toyplot barplot of the results table


Instruction:
Can you tell what is the following Python 3 function doing
def plot(self, pcs=[1, 2], ax=None, cmap=None, cdict=None, legend=True, title=None, outfile=None):
        ## Specify which 2 pcs to plot, default is pc1 and pc2
        pc1 = pcs[0] - 1
        pc2 = pcs[1] - 1
        if pc1 < 0 or pc2 > self.ncomponents - 1:
            raise IPyradError("PCs are 1-indexed. 1 is min & {} is max".format(self.ncomponents))

        ## Convert genotype data to allele count data
        ## We do this here because we might want to try different ways
        ## of accounting for missing data and "alt" allele counts treat
        ## missing data as "ref"
        allele_counts = self.genotypes.to_n_alt()

        ## Actually do the pca
        if self.ncomponents > len(self.samples_vcforder):
            self.ncomponents = len(self.samples_vcforder)
            print("  INFO: # PCs < # samples. Forcing # PCs = {}".format(self.ncomponents))
        coords, model = allel.pca(allele_counts, n_components=self.ncomponents, scaler='patterson')

        self.pcs = pd.DataFrame(coords,
                                index=self.samples_vcforder,
                                columns=["PC{}".format(x) for x in range(1,self.ncomponents+1)])

        ## Just allow folks to pass in the name of the cmap they want to use
        if isinstance(cmap, str):
            try:
                cmap = cm.get_cmap(cmap)
            except:
                raise IPyradError("  Bad cmap value: {}".format(cmap))

        if not cmap and not cdict:
            if not self.quiet:
                print("  Using default cmap: Spectral")
            cmap = cm.get_cmap('Spectral')

        if cmap:
            if cdict:
                print("  Passing in both cmap and cdict defaults to using the cmap value.")
            popcolors = cmap(np.arange(len(self.pops))/len(self.pops))
            cdict = {i:j for i, j in zip(self.pops.keys(), popcolors)}

        fig = ""
        if not ax:
            fig = plt.figure(figsize=(6, 5))
            ax = fig.add_subplot(1, 1, 1)

        x = coords[:, pc1]
        y = coords[:, pc2]
        for pop in self.pops:
            ## Don't include pops with no samples, it makes the legend look stupid
            ## TODO: This doesn't prevent empty pops from showing up in the legend for some reason.
            if len(self.pops[pop]) > 0:
                mask = np.isin(self.samples_vcforder, self.pops[pop])
                ax.plot(x[mask], y[mask], marker='o', linestyle=' ', color=cdict[pop], label=pop, markersize=6, mec='k', mew=.5)

        ax.set_xlabel('PC%s (%.1f%%)' % (pc1+1, model.explained_variance_ratio_[pc1]*100))
        ax.set_ylabel('PC%s (%.1f%%)' % (pc2+1, model.explained_variance_ratio_[pc2]*100))

        if legend:
            ax.legend(bbox_to_anchor=(1, 1), loc='upper left')

        if fig:
            fig.tight_layout()

        if title:
            ax.set_title(title)

        if outfile:
            try:
                plt.savefig(outfile, format="png", bbox_inches="tight")
            except:
                print("  Saving pca.plot() failed to save figure to {}".format(outfile))

        return ax

Response: Plot the PCA and plot it.


Instruction:
Make a summary of the following Python 3 code
def plot_pairwise_dist(self, labels=None, ax=None, cmap=None, cdict=None, metric="euclidean"):
        allele_counts = self.genotypes.to_n_alt()
        dist = allel.pairwise_distance(allele_counts, metric=metric)
        if not ax:
            fig = plt.figure(figsize=(5, 5))
            ax = fig.add_subplot(1, 1, 1)

        if isinstance(labels, bool):
            if labels:
                labels = list(self.samples_vcforder)
        elif isinstance(labels, type(None)):
            pass
        else:
            ## If not bool or None (default), then check to make sure the list passed in
            ## is the right length
            if not len(labels) == len(self.samples_vcforder):
                raise IPyradError(LABELS_LENGTH_ERROR.format(len(labels), len(self.samples_vcforder)))

        allel.plot.pairwise_distance(dist, labels=labels, ax=ax, colorbar=False)

Response: Plot pairwise distance between all samples in the species.


Instruction:
Write a Python 3 script for
returning a copy of the pca analysis object

Response:
def copy(self):
        """ returns a copy of the pca analysis object """
        cp = copy.deepcopy(self)
        cp.genotypes = allel.GenotypeArray(self.genotypes, copy=True)
        return cp


Instruction:
Create a Python 3 function for
converting ipyrad. loci file to iqtree - pomo counts file.

Response:
def loci2cf(name, locifile, popdict, wdir=None, ipyclient=None):
    """
    Convert ipyrad .loci file to an iqtree-pomo 'counts' file

    Parameters:
    -----------
    name:
        A prefix name for output files that will be produced
    locifile:
        A .loci file produced by ipyrad.
    popdict:
        A python dictionary grouping Clade names to Sample names.
        Example: {"A": ['a', 'b', 'c'], "B": ['d', 'e', 'f']}
    ipyclient:
        If you pass it an ipyclient it will distribute work over
        remote engines, otherwise we use multiprocessing (todo).
    """

    ## working directory, make sure it exists
    if wdir:
        wdir = os.path.abspath(wdir)
        if not os.path.exists(wdir):
            raise IPyradWarningExit(" working directory (wdir) does not exist")
    else:
        wdir = os.path.curdir

    ## output file path
    name = name.rsplit(".cf")[0]
    outfile = os.path.join(wdir, "{}.cf".format(name))
    out = open(outfile, 'w')

    ## parse loci file
    with open(locifile) as inloc:
        loci = inloc.read().strip().split("|\n")

    ## get all names
    names = list(itertools.chain(*popdict.values()))
    popkeys = sorted(popdict.keys())

    ## count nsites
    nsites = sum(len(loc.split("\n")[0].split()[1]) for loc in loci[:])

    ## print the header
    out.write(HEADER.format(**{"NPOP": len(popdict),
                               "NSITES": nsites,
                               "VTAXA": "\t".join(popkeys)}))

    ## build print string
    outstr = "chr{:<8}  {:<4}  "
    for cidx in xrange(len(popkeys)):
        outstr += "{:<8}  "

    toprint = []
    for idx in xrange(len(loci)):
        dat = loci[idx].split("\n")
        seqs = np.array([list(i.split()[1]) for i in dat[:-1]])
        names = [i.split()[0] for i in dat[:-1]]
        data = np.zeros((seqs.shape[1], len(popkeys), 4), dtype=np.uint16)

        for sidx in xrange(seqs.shape[1]):
            for cidx in xrange(len(popkeys)):
                for name in popdict[popkeys[cidx]]:
                    if name in names:
                        base = seqs[names.index(name), sidx]
                        if base in list("ACGT"):
                            data[sidx, cidx, BASE2IDX[base]] += 2
                        elif base in list("RSYMKW"):
                            base1, base2 = AMBIGS[base]
                            data[sidx, cidx, BASE2IDX[base1]] += 1
                            data[sidx, cidx, BASE2IDX[base2]] += 1

            ## print string for one locus
            sdat = [",".join([str(i) for i in i.tolist()]) for i in data[sidx]]
            #print outstr.format(idx+1, sidx+1, *sdat)
            toprint.append(outstr.format(idx+1, sidx+1, *sdat))

        ## if 10K loci, then print and clear
        if not idx % 10000:
            out.write("\n".join(toprint)+"\n")
            toprint = []

    ## close handle
    out.write("\n".join(toprint)+"\n")
    out.close()


Instruction:
Can you generate a brief explanation for the following Python 3 code
def update(assembly, idict, count):

    data = iter(open(os.path.join(assembly.dirs.outfiles,
                     assembly.name+".phy"), 'r'))

    ntax, nchar = data.next().strip().split()

    ## read in max N bp at a time
    for line in data:
        tax, seq = line.strip().split()
        idict[tax] = idict[tax][100000:]
        idict[tax] += seq[count:count+100000]
    del line

    return idict

Response:
updates dictionary with the next. 5M reads from the super long string
    phylip file. Makes faster reading.


Instruction:
Create a Python 3 function to
build a phy file from a list of samples.

Response:
def makephy(data, samples, longname):
    """ builds phy output. If large files writes 50000 loci at a time to tmp
    files and rebuilds at the end"""

    ## order names
    names = [i.name for i in samples]
    names.sort()

    ## read in loci file
    locifile = os.path.join(data.dirs.outfiles, data.name+".loci")
    locus = iter(open(locifile, 'rb'))

    ## dict for saving the full matrix
    fdict = {name:[] for name in names}

    ## list for saving locus number and locus range for partitions
    partitions = []
    initial_pos = 1

    ## remove empty column sites and append edited seqs to dict F
    done = 0
    nloci = 0
    nbases = 0

    ## TODO: This should be fixed. it cycles through reading each locus
    ## until nloci is less than this large number. It should really just
    ## read to the end of the file, so it'll do all loci no matter how
    ## many there are.
    while nloci < 5000000:
        seqs = []
        #arrayed = np.array([])
        anames = []
        while 1:
            ## get next locus
            try:
                samp = locus.next()
            except StopIteration:
                done = 1
                break
            if "//" in samp:
                nloci += 1
                break
            else:
                try:
                    name, seq = samp.split()
                except ValueError:
                    print samp
                anames.append(name[1:])
                seqs.append(seq.strip())
        ## reset
        arrayed = np.array([list(i) for i in seqs])
        if done:
            break
        ## create mask for columns that are empty or
        ## that are paired-end separators (compatible w/ pyrad v2 and v3)
        #mask = [i for i in range(len(arrayed.T)) if np.any([
        ## still surely a better way to vectorize this...
        mask = [i for i in arrayed.T if any([j not in list("-Nn") for j in i])]
        masked = np.dstack(mask)[0]

        ## partition information
        loc_name = "p"+str(nloci)
        loc_range = str(initial_pos) + "-" +\
                    str(len(masked[0]) + initial_pos -1)
        initial_pos += len(masked[0])
        partitions.append(loc_name+"="+loc_range)

        ## uncomment to print block info (used to partition by locus)
        #blockend += minray
        #print blockend,
        #print loc
        #print arrayed

        ## append data to dict
        for name in names:
            if name in anames:
                #fdict[name].append(arrayed[anames.index(name), mask].tostring())
                fdict[name].append(masked[anames.index(name),:].tostring())
            else:
                fdict[name].append("N"*masked.shape[1])
                #fdict[name].append("N"*len(arrayed[0, mask]))
        ## add len to total length
        nbases += len(fdict[name][-1])

        ## after x iterations tmp pickle fdict?
        if not nloci % 1e4:
            ## concat strings
            for name in fdict:
                with open(os.path.join(assembly.dirs.outfiles , "tmp",
                    "{}_{}.phy.tmp".format(name, nloci)), 'wb') as wout:
                    wout.write("".join(fdict[name]))
            del fdict
            fdict = {name:[] for name in names}

    ## print out .PHY file, if really big, pull form multiple tmp pickle
    superout = open(os.path.join( assembly.dirs.outfiles, assembly.name+".phy" ), 'wb')
    print >>superout, len(names), nbases
    if nloci < 1e4:
        for name in names:
            print >>superout, name+(" "*((longname+3)-\
                              len(name)))+"".join(fdict[name])
    else:
        for name in names:
            superout.write("{}{}{}".format(
                            name,
                            " "*((longname+3)-len(name)),
                            "".join(fdict[name])))
            tmpfiles = glob.glob(os.path.join(assembly.dirs.outfiles, "tmp", name+"*.phy.tmp"))
            tmpfiles.sort()
            for tmpf in tmpfiles:
                with open(tmpf, 'rb') as tmpin:
                    superout.write(tmpin.read())
                os.remove(tmpf)
            superout.write("\n")
    superout.close()
    raxml_part_out = open(os.path.join(assembly.dirs.outfiles, assembly.name+".phy.partitions"), 'w')
    for partition in partitions:
        print >>raxml_part_out, "DNA, %s" % (partition)
    raxml_part_out.close()

    return partitions


Instruction:
How would you implement a function in Python 3 that
makes a phylip and nexus format for a list of samples.

Response:
def make(assembly, samples):
    """ Make phylip and nexus formats. This is hackish since I'm recycling the
    code whole-hog from pyrad V3. Probably could be good to go back through
    and clean up the conversion code some time.
    """

    ## get the longest name
    longname = max([len(i) for i in assembly.samples.keys()])
    names = [i.name for i in samples]

    partitions = makephy(assembly, samples, longname)
    makenex(assembly, names, longname, partitions)


Instruction:
Can you write a function in Python 3 where it
cleans up a bunch of loose files.

Response:
def sample_cleanup(data, sample):
    """
    Clean up a bunch of loose files.
    """
    umap1file = os.path.join(data.dirs.edits, sample.name+"-tmp-umap1.fastq")
    umap2file = os.path.join(data.dirs.edits, sample.name+"-tmp-umap2.fastq")
    unmapped = os.path.join(data.dirs.refmapping, sample.name+"-unmapped.bam")
    samplesam = os.path.join(data.dirs.refmapping, sample.name+".sam")
    split1 = os.path.join(data.dirs.edits, sample.name+"-split1.fastq")
    split2 = os.path.join(data.dirs.edits, sample.name+"-split2.fastq")
    refmap_derep = os.path.join(data.dirs.edits, sample.name+"-refmap_derep.fastq")
    for f in [umap1file, umap2file, unmapped, samplesam, split1, split2, refmap_derep]:
        try:
            os.remove(f)
        except:
            pass


Instruction:
Can you create a Python 3 function that
indexes the reference sequence.

Response:
def index_reference_sequence(data, force=False):
    """
    Index the reference sequence, unless it already exists. Also make a mapping
    of scaffolds to index numbers for later user in steps 5-6.
    """

    ## get ref file from params
    refseq_file = data.paramsdict['reference_sequence']
    index_files = []

    ## Check for existence of index files. Default to bwa unless you specify smalt
    if "smalt" in data._hackersonly["aligner"]:
        # These are smalt index files. Only referenced here to ensure they exist
        index_files.extend([".sma", ".smi"])
    else:
        index_files.extend([".amb", ".ann", ".bwt", ".pac", ".sa"])

    ## samtools specific index
    index_files.extend([".fai"])

    ## If reference sequence already exists then bail out of this func
    if not force:
        if all([os.path.isfile(refseq_file+i) for i in index_files]):
            return
    #if data._headers:
    #    print(INDEX_MSG.format(data._hackersonly["aligner"]))

    if "smalt" in data._hackersonly["aligner"]:
        ## Create smalt index for mapping
        ## smalt index [-k <wordlen>] [-s <stepsiz>]  <index_name> <reference_file>
        cmd1 = [ipyrad.bins.smalt, "index",
                "-k", str(data._hackersonly["smalt_index_wordlen"]),
                refseq_file,
                refseq_file]
    else:
        ## bwa index <reference_file>
        cmd1 = [ipyrad.bins.bwa, "index", refseq_file]

    ## call the command
    LOGGER.info(" ".join(cmd1))
    proc1 = sps.Popen(cmd1, stderr=sps.STDOUT, stdout=sps.PIPE)
    error1 = proc1.communicate()[0]

    ## simple samtools index for grabbing ref seqs
    cmd2 = [ipyrad.bins.samtools, "faidx", refseq_file]
    LOGGER.info(" ".join(cmd2))
    proc2 = sps.Popen(cmd2, stderr=sps.STDOUT, stdout=sps.PIPE)
    error2 = proc2.communicate()[0]

    ## error handling
    if proc1.returncode:
        raise IPyradWarningExit(error1)
    if error2:
        if "please use bgzip" in error2:
            raise IPyradWarningExit(NO_ZIP_BINS.format(refseq_file))
        else:
            raise IPyradWarningExit(error2)


Instruction:
Implement a Python 3 function for
mapping reads to reference sequence.

Response:
def mapreads(data, sample, nthreads, force):
    """
    Attempt to map reads to reference sequence. This reads in the fasta files
    (samples.files.edits), and maps each read to the reference. Unmapped reads
    are dropped right back in the de novo pipeline. Reads that map successfully
    are processed and pushed downstream and joined with the rest of the data
    post muscle_align.

    Mapped reads end up in a sam file.
    """

    LOGGER.info("Entering mapreads(): %s %s", sample.name, nthreads)

    ## This is the input derep file, for paired data we need to split the data,
    ## and so we will make sample.files.dereps == [derep1, derep2], but for
    ## SE data we can simply use sample.files.derep == [derepfile].
    derepfile = os.path.join(data.dirs.edits, sample.name+"_derep.fastq")
    sample.files.dereps = [derepfile]

    ## This is the final output files containing merged/concat derep'd refmap'd
    ## reads that did not match to the reference. They will be back in
    ## merge/concat (--nnnnn--) format ready to be input to vsearch, if needed.
    mumapfile = sample.files.unmapped_reads
    umap1file = os.path.join(data.dirs.edits, sample.name+"-tmp-umap1.fastq")
    umap2file = os.path.join(data.dirs.edits, sample.name+"-tmp-umap2.fastq")

    ## split the derepfile into the two handles we designate
    if "pair" in data.paramsdict["datatype"]:
        sample.files.split1 = os.path.join(data.dirs.edits, sample.name+"-split1.fastq")
        sample.files.split2 = os.path.join(data.dirs.edits, sample.name+"-split2.fastq")
        sample.files.dereps = [sample.files.split1, sample.files.split2]
        split_merged_reads(sample.files.dereps, derepfile)

    ## (cmd1) smalt <task> [TASK_OPTIONS] [<index_name> <file_name_A> [<file_name_B>]]
    ##  -f sam       : Output as sam format, tried :clip: to hard mask output
    ##                 but it shreds the unmapped reads (outputs empty fq)
    ##  -l [pe,mp,pp]: If paired end select the orientation of each read
    ##  -n #         : Number of threads to use
    ##  -x           : Perform a more exhaustive search
    ##  -y #         : proportion matched to reference (sequence similarity)
    ##  -o           : output file
    ##               : Reference sequence
    ##               : Input file(s), in a list. One for R1 and one for R2
    ##  -c #         : proportion of the query read length that must be covered

    ## (cmd1) bwa mem [OPTIONS] <index_name> <file_name_A> [<file_name_B>] > <output_file>
    ##  -t #         : Number of threads
    ##  -M           : Mark split alignments as secondary.

    ## (cmd2) samtools view [options] <in.bam>|<in.sam>|<in.cram> [region ...]
    ##   -b = write to .bam
    ##   -q = Only keep reads with mapq score >= 30 (seems to be pretty standard)
    ##   -F = Select all reads that DON'T have these flags.
    ##         0x4 (segment unmapped)
    ##         0x100 (Secondary alignment)
    ##         0x800 (supplementary alignment)
    ##   -U = Write out all reads that don't pass the -F filter
    ##        (all unmapped reads go to this file).

    ## TODO: Should eventually add `-q 13` to filter low confidence mapping.
    ## If you do this it will throw away some fraction of reads. Ideally you'd
    ## catch these and throw them in with the rest of the unmapped reads, but
    ## I can't think of a straightforward way of doing that. There should be
    ## a `-Q` flag to only keep reads below the threshold, but i realize that
    ## would be of limited use besides for me.

    ## (cmd3) samtools sort [options...] [in.bam]
    ##   -T = Temporary file name, this is required by samtools, ignore it
    ##        Here we hack it to be samhandle.tmp cuz samtools cleans it up
    ##   -O = Output file format, in this case bam
    ##   -o = Output file name

    if "smalt" in data._hackersonly["aligner"]:
        ## The output SAM data is written to file (-o)
        ## input is either (derep) or (derep-split1, derep-split2)
        cmd1 = [ipyrad.bins.smalt, "map",
                "-f", "sam",
                "-n", str(max(1, nthreads)),
                "-y", str(data.paramsdict['clust_threshold']),
                "-o", os.path.join(data.dirs.refmapping, sample.name+".sam"),
                "-x",
                data.paramsdict['reference_sequence']
                ] + sample.files.dereps
        cmd1_stdout = sps.PIPE
        cmd1_stderr = sps.STDOUT
    else:
        cmd1 = [ipyrad.bins.bwa, "mem",
                "-t", str(max(1, nthreads)),
                "-M",
                data.paramsdict['reference_sequence']
                ] + sample.files.dereps
        ## Insert optional flags for bwa
        try:
            bwa_args = data._hackersonly["bwa_args"].split()
            bwa_args.reverse()
            for arg in bwa_args:
                cmd1.insert(2, arg)
        except KeyError:
            ## Do nothing
            pass
        cmd1_stdout = open(os.path.join(data.dirs.refmapping, sample.name+".sam"), 'w')
        cmd1_stderr = None

    ## Reads in the SAM file from cmd1. It writes the unmapped data to file
    ## and it pipes the mapped data to be used in cmd3
    cmd2 = [ipyrad.bins.samtools, "view",
           "-b",
           ## TODO: This introduces a bug with PE right now. Think about the case where
           ## R1 has low qual mapping and R2 has high. You get different numbers
           ## of reads in the unmapped tmp files. FML.
           #"-q", "30",
           "-F", "0x904",
           "-U", os.path.join(data.dirs.refmapping, sample.name+"-unmapped.bam"),
           os.path.join(data.dirs.refmapping, sample.name+".sam")]

    ## this is gonna catch mapped bam output from cmd2 and write to file
    cmd3 = [ipyrad.bins.samtools, "sort",
            "-T", os.path.join(data.dirs.refmapping, sample.name+".sam.tmp"),
            "-O", "bam",
            "-o", sample.files.mapped_reads]

    ## TODO: Unnecessary?
    ## this is gonna read the sorted BAM file and index it. only for pileup?
    cmd4 = [ipyrad.bins.samtools, "index", sample.files.mapped_reads]

    ## this is gonna read in the unmapped files, args are added below,
    ## and it will output fastq formatted unmapped reads for merging.
    ## -v 45 sets the default qscore arbitrarily high
    cmd5 = [ipyrad.bins.samtools, "bam2fq", "-v 45",
            os.path.join(data.dirs.refmapping, sample.name+"-unmapped.bam")]

    ## Insert additional arguments for paired data to the commands.
    ## We assume Illumina paired end reads for the orientation
    ## of mate pairs (orientation: ---> <----).
    if 'pair' in data.paramsdict["datatype"]:
        if "smalt" in data._hackersonly["aligner"]:
            ## add paired flag (-l pe) to cmd1 right after (smalt map ...)
            cmd1.insert(2, "pe")
            cmd1.insert(2, "-l")
        else:
            ## No special PE flags for bwa
            pass
        ## add samtools filter for only keep if both pairs hit
        ## 0x1 - Read is paired
        ## 0x2 - Each read properly aligned
        cmd2.insert(2, "0x3")
        cmd2.insert(2, "-f")

        ## tell bam2fq that there are output files for each read pair
        cmd5.insert(2, umap1file)
        cmd5.insert(2, "-1")
        cmd5.insert(2, umap2file)
        cmd5.insert(2, "-2")
    else:
        cmd5.insert(2, mumapfile)
        cmd5.insert(2, "-0")

    ## Running cmd1 creates ref_mapping/sname.sam,
    LOGGER.debug(" ".join(cmd1))
    proc1 = sps.Popen(cmd1, stderr=cmd1_stderr, stdout=cmd1_stdout)

    ## This is really long running job so we wrap it to ensure it dies.
    try:
        error1 = proc1.communicate()[0]
    except KeyboardInterrupt:
        proc1.kill()

    ## raise error if one occurred in smalt
    if proc1.returncode:
        raise IPyradWarningExit(error1)

    ## Running cmd2 writes to ref_mapping/sname.unmapped.bam, and
    ## fills the pipe with mapped BAM data
    LOGGER.debug(" ".join(cmd2))
    proc2 = sps.Popen(cmd2, stderr=sps.STDOUT, stdout=sps.PIPE)

    ## Running cmd3 pulls mapped BAM from pipe and writes to
    ## ref_mapping/sname.mapped-sorted.bam.
    ## Because proc2 pipes to proc3 we just communicate this to run both.
    LOGGER.debug(" ".join(cmd3))
    proc3 = sps.Popen(cmd3, stderr=sps.STDOUT, stdout=sps.PIPE, stdin=proc2.stdout)
    error3 = proc3.communicate()[0]
    if proc3.returncode:
        raise IPyradWarningExit(error3)
    proc2.stdout.close()

    ## Later we're gonna use samtools to grab out regions using 'view', and to
    ## do that we need it to be indexed. Let's index it now.
    LOGGER.debug(" ".join(cmd4))
    proc4 = sps.Popen(cmd4, stderr=sps.STDOUT, stdout=sps.PIPE)
    error4 = proc4.communicate()[0]
    if proc4.returncode:
        raise IPyradWarningExit(error4)

    ## Running cmd5 writes to either edits/sname-refmap_derep.fastq for SE
    ## or it makes edits/sname-tmp-umap{12}.fastq for paired data, which
    ## will then need to be merged.
    LOGGER.debug(" ".join(cmd5))
    proc5 = sps.Popen(cmd5, stderr=sps.STDOUT, stdout=sps.PIPE)
    error5 = proc5.communicate()[0]
    if proc5.returncode:
        raise IPyradWarningExit(error5)

    ## Finally, merge the unmapped reads, which is what cluster()
    ## expects. If SE, just rename the outfile. In the end
    ## <sample>-refmap_derep.fq will be the final output
    if 'pair' in data.paramsdict["datatype"]:
        LOGGER.info("Merging unmapped reads {} {}".format(umap1file, umap2file))
        merge_pairs_after_refmapping(data, [(umap1file, umap2file)], mumapfile)


Instruction:
Write a Python 3 function that can
build a single end cluster from the refmapped data.

Response:
def fetch_cluster_se(data, samfile, chrom, rstart, rend):
    """
    Builds a single end cluster from the refmapped data.
    """

    ## If SE then we enforce the minimum overlap distance to avoid the
    ## staircase syndrome of multiple reads overlapping just a little.
    overlap_buffer = data._hackersonly["min_SE_refmap_overlap"]

    ## the *_buff variables here are because we have to play patty
    ## cake here with the rstart/rend vals because we want pysam to
    ## enforce the buffer for SE, but we want the reference sequence
    ## start and end positions to print correctly for downstream.
    rstart_buff = rstart + overlap_buffer
    rend_buff = rend - overlap_buffer

    ## Reads that map to only very short segements of the reference
    ## sequence will return buffer end values that are before the
    ## start values causing pysam to complain. Very short mappings.
    if rstart_buff > rend_buff:
        tmp = rstart_buff
        rstart_buff = rend_buff
        rend_buff = tmp
    ## Buffering can't make start and end equal or pysam returns nothing.
    if rstart_buff == rend_buff:
        rend_buff += 1

    ## store pairs
    rdict = {}
    clust = []
    iterreg = []

    iterreg = samfile.fetch(chrom, rstart_buff, rend_buff)

    ## use dict to match up read pairs
    for read in iterreg:
        if read.qname not in rdict:
            rdict[read.qname] = read

    ## sort dict keys so highest derep is first ('seed')
    sfunc = lambda x: int(x.split(";size=")[1].split(";")[0])
    rkeys = sorted(rdict.keys(), key=sfunc, reverse=True)

    ## get blocks from the seed for filtering, bail out if seed is not paired
    try:
        read1 = rdict[rkeys[0]]
    except ValueError:
        LOGGER.error("Found bad cluster, skipping - key:{} rdict:{}".format(rkeys[0], rdict))
        return ""

    ## the starting blocks for the seed
    poss = read1.get_reference_positions(full_length=True)
    seed_r1start = min(poss)
    seed_r1end = max(poss)

    ## store the seed -------------------------------------------
    if read1.is_reverse:
        seq = revcomp(read1.seq)
    else:
        seq = read1.seq

    ## store, could write orient but just + for now.
    size = sfunc(rkeys[0])
    clust.append(">{}:{}:{};size={};*\n{}"\
        .format(chrom, seed_r1start, seed_r1end, size, seq))

    ## If there's only one hit in this region then rkeys will only have
    ## one element and the call to `rkeys[1:]` will raise. Test for this.
    if len(rkeys) > 1:
        ## store the hits to the seed -------------------------------
        for key in rkeys[1:]:
            skip = False
            try:
                read1 = rdict[key]
            except ValueError:
                ## enter values that will make this read get skipped
                read1 = rdict[key][0]
                skip = True

            ## orient reads only if not skipping
            if not skip:
                poss = read1.get_reference_positions(full_length=True)
                minpos = min(poss)
                maxpos = max(poss)
                ## store the seq
                if read1.is_reverse:
                    seq = revcomp(read1.seq)
                else:
                    seq = read1.seq
                ## store, could write orient but just + for now.
                size = sfunc(key)
                clust.append(">{}:{}:{};size={};+\n{}"\
                    .format(chrom, minpos, maxpos, size, seq))
            else:
                ## seq is excluded, though, we could save it and return
                ## it as a separate cluster that will be aligned separately.
                pass

    return clust


Instruction:
Make a summary of the following Python 3 code
def fetch_cluster_pairs(data, samfile, chrom, rstart, rend):
    ## store pairs
    rdict = {}
    clust = []

    ## grab the region and make tuples of info
    iterreg = samfile.fetch(chrom, rstart, rend)

    ## use dict to match up read pairs
    for read in iterreg:
        if read.qname not in rdict:
            rdict[read.qname] = [read]
        else:
            rdict[read.qname].append(read)

    ## sort dict keys so highest derep is first ('seed')
    sfunc = lambda x: int(x.split(";size=")[1].split(";")[0])
    rkeys = sorted(rdict.keys(), key=sfunc, reverse=True)

    ## get blocks from the seed for filtering, bail out if seed is not paired
    try:
        read1, read2 = rdict[rkeys[0]]
    except ValueError:
        return 0

    ## the starting blocks for the seed
    poss = read1.get_reference_positions() + read2.get_reference_positions()
    seed_r1start = min(poss)
    seed_r2end = max(poss)

    ## store the seed -------------------------------------------
    ## Simplify. R1 and R2 are always on opposite strands, but the
    ## orientation is variable. We revcomp and order the reads to
    ## preserve genomic order.
    reads_overlap = False
    if read1.is_reverse:
        if read2.aend > read1.get_blocks()[0][0]:
            reads_overlap = True
            seq = read2.seq + "nnnn" + revcomp(read1.seq)
        else:
            seq = read2.seq + "nnnn" + read1.seq
    else:
        if read1.aend > read2.get_blocks()[0][0]:
            reads_overlap = True
            seq = read1.seq + "nnnn" + revcomp(read2.seq)
        else:
            seq = read1.seq + "nnnn" + read2.seq

    ## store, could write orient but just + for now.
    size = sfunc(rkeys[0])
    clust.append(">{}:{}:{};size={};*\n{}"\
        .format(chrom, seed_r1start, seed_r2end, size, seq))

    ## If there's only one hit in this region then rkeys will only have
    ## one element and the call to `rkeys[1:]` will raise. Test for this.
    if len(rkeys) > 1:
        ## store the hits to the seed -------------------------------
        for key in rkeys[1:]:
            skip = False
            try:
                read1, read2 = rdict[key]
            except ValueError:
                ## enter values that will make this read get skipped
                read1 = rdict[key][0]
                read2 = read1
                skip = True

            ## orient reads and filter out ones that will not align well b/c
            ## they do not overlap enough with the seed
            poss = read1.get_reference_positions() + read2.get_reference_positions()
            minpos = min(poss)
            maxpos = max(poss)

            ## skip if more than one hit location
            if read1.has_tag("SA") or read2.has_tag("SA"):
                skip = True

            ## store if read passes
            if (abs(minpos - seed_r1start) < 50) and \
               (abs(maxpos - seed_r2end) < 50) and \
               (not skip):
                ## store the seq
                if read1.is_reverse:
                    ## do reads overlap
                    if read2.aend > read1.get_blocks()[0][0]:
                        reads_overlap = True
                        seq = read2.seq + "nnnn" + revcomp(read1.seq)
                    else:
                        seq = read2.seq + "nnnn" + read1.seq
                else:
                    if read1.aend > read2.get_blocks()[0][0]:
                        reads_overlap = True
                        seq = read1.seq + "nnnn" + revcomp(read2.seq)
                    else:
                        seq = read1.seq + "nnnn" + read2.seq

                ## store, could write orient but just + for now.
                size = sfunc(key)
                clust.append(">{}:{}:{};size={};+\n{}"\
                    .format(chrom, minpos, maxpos, size, seq))
            else:
                ## seq is excluded, though, we could save it and return
                ## it as a separate cluster that will be aligned separately.
                pass

    ## merge the pairs prior to returning them
    ## Remember, we already tested for quality scores, so
    ## merge_after_pysam will generate arbitrarily high scores
    ## It would be nice to do something here like test if
    ## the average insert length + 2 stdv is > 2*read len
    ## so you can switch off merging for mostly non-overlapping data
    if reads_overlap:
        if data._hackersonly["refmap_merge_PE"]:
            clust = merge_after_pysam(data, clust)
            #clust = merge_pair_pipes(data, clust)

    return clust

Response: Builds a paired cluster from the refmapped data.


Instruction:
Write a Python 3 function for
building a chunk for each region of a sample and add it to the list of clusters.

Response:
def ref_build_and_muscle_chunk(data, sample):
    """
    1. Run bedtools to get all overlapping regions
    2. Parse out reads from regions using pysam and dump into chunk files.
       We measure it out to create 10 chunk files per sample.
    3. If we really wanted to speed this up, though it is pretty fast already,
       we could parallelize it since we can easily break the regions into
       a list of chunks.
    """

    ## get regions using bedtools
    regions = bedtools_merge(data, sample).strip().split("\n")
    nregions = len(regions)
    chunksize = (nregions / 10) + (nregions % 10)

    LOGGER.debug("nregions {} chunksize {}".format(nregions, chunksize))
    ## create an output file to write clusters to
    idx = 0
    tmpfile = os.path.join(data.tmpdir, sample.name+"_chunk_{}.ali")
    ## remove old files if they exist to avoid append errors
    for i in range(11):
        if os.path.exists(tmpfile.format(i)):
            os.remove(tmpfile.format(i))
    fopen = open

    ## If reference+denovo we drop the reads back into clust.gz
    ## and let the muscle_chunker do it's thing back in cluster_within
    if data.paramsdict["assembly_method"] == "denovo+reference":
        tmpfile = os.path.join(data.dirs.clusts, sample.name+".clust.gz")
        fopen = gzip.open

    ## build clusters for aligning with muscle from the sorted bam file
    samfile = pysam.AlignmentFile(sample.files.mapped_reads, 'rb')
    #"./tortas_refmapping/PZ70-mapped-sorted.bam", "rb")

    ## fill clusts list and dump periodically
    clusts = []
    nclusts = 0
    for region in regions:
        chrom, pos1, pos2 = region.split()
        try:
            ## fetches pairs quickly but then goes slow to merge them.
            if "pair" in data.paramsdict["datatype"]:
                clust = fetch_cluster_pairs(data, samfile, chrom, int(pos1), int(pos2))

            ## fetch but no need to merge
            else:
                clust = fetch_cluster_se(data, samfile, chrom, int(pos1), int(pos2))
        except IndexError as inst:
            LOGGER.error("Bad region chrom:start-end {}:{}-{}".format(chrom, pos1, pos2))
            continue
        if clust:
            clusts.append("\n".join(clust))
            nclusts += 1

            if nclusts == chunksize:
                ## write to file
                tmphandle = tmpfile.format(idx)
                with fopen(tmphandle, 'a') as tmp:
                    #LOGGER.debug("Writing tmpfile - {}".format(tmpfile.format(idx)))
                    #if data.paramsdict["assembly_method"] == "denovo+reference":
                    #    ## This is dumb, but for this method you need to prepend the
                    #    ## separator to maintain proper formatting of clust.gz
                    tmp.write("\n//\n//\n".join(clusts)+"\n//\n//\n")
                idx += 1
                nclusts = 0
                clusts = []
    if clusts:
        ## write remaining to file
        with fopen(tmpfile.format(idx), 'a') as tmp:
            #tmp.write("\n//\n//\n" + ("\n//\n//\n".join(clusts)))
            tmp.write("\n//\n//\n".join(clusts)+"\n//\n//\n")
        clusts = []

    if not data.paramsdict["assembly_method"] == "denovo+reference":
        chunkfiles = glob.glob(os.path.join(data.tmpdir, sample.name+"_chunk_*.ali"))
        LOGGER.info("created chunks %s", chunkfiles)

    ## cleanup
    samfile.close()


Instruction:
Can you write a function in Python 3 where it
runs bedtools to get all overlapping regions. Pass this list into the func 'get_overlapping_reads' which will write fastq chunks to the clust.gz file. 1) Run bedtools merge to get a list of all contiguous blocks of bases in the reference seqeunce where one or more of our reads overlap. The output will look like this: 1 45230754 45230783 1 74956568 74956596 ... 1 116202035 116202060

Response:
def ref_muscle_chunker(data, sample):
    """
    Run bedtools to get all overlapping regions. Pass this list into the func
    'get_overlapping_reads' which will write fastq chunks to the clust.gz file.

    1) Run bedtools merge to get a list of all contiguous blocks of bases
    in the reference seqeunce where one or more of our reads overlap.
    The output will look like this:
            1       45230754        45230783
            1       74956568        74956596
            ...
            1       116202035       116202060
    """
    LOGGER.info('entering ref_muscle_chunker')

    ## Get regions, which will be a giant list of 5-tuples, of which we're
    ## only really interested in the first three: (chrom, start, end) position.
    regions = bedtools_merge(data, sample)

    if len(regions) > 0:
        ## this calls bam_region_to_fasta a billion times
        get_overlapping_reads(data, sample, regions)
    else:
        msg = "No reads mapped to reference sequence - {}".format(sample.name)
        LOGGER.warn(msg)


Instruction:
How would you code a function in Python 3 to
take merged file from vsearch derep and splits it into R1 and R2 parts.

Response:
def split_merged_reads(outhandles, input_derep):
    """
    Takes merged/concat derep file from vsearch derep and split it back into
    separate R1 and R2 parts.
    - sample_fastq: a list of the two file paths to write out to.
    - input_reads: the path to the input merged reads
    """

    handle1, handle2 = outhandles
    splitderep1 = open(handle1, 'w')
    splitderep2 = open(handle2, 'w')

    with open(input_derep, 'r') as infile:
        ## Read in the infile two lines at a time: (seqname, sequence)
        duo = itertools.izip(*[iter(infile)]*2)

        ## lists for storing results until ready to write
        split1s = []
        split2s = []

        ## iterate over input splitting, saving, and writing.
        idx = 0
        while 1:
            try:
                itera = duo.next()
            except StopIteration:
                break
            ## split the duo into separate parts and inc counter
            part1, part2 = itera[1].split("nnnn")
            idx += 1

            ## R1 needs a newline, but R2 inherits it from the original file
            ## store parts in lists until ready to write
            split1s.append("{}{}\n".format(itera[0], part1))
            split2s.append("{}{}".format(itera[0], part2))

            ## if large enough then write to file
            if not idx % 10000:
                splitderep1.write("".join(split1s))
                splitderep2.write("".join(split2s))
                split1s = []
                split2s = []

    ## write final chunk if there is any
    if any(split1s):
        splitderep1.write("".join(split1s))
        splitderep2.write("".join(split2s))

    ## close handles
    splitderep1.close()
    splitderep2.close()


Instruction:
How would you code a function in Python 3 to
check mean insert size for this sample and update hackersonly. max_inner_mate_distance controls how far apart mate pairs can be considered for bedtools merging downstream.

Response:
def check_insert_size(data, sample):
    """
    check mean insert size for this sample and update
    hackersonly.max_inner_mate_distance if need be. This value controls how
    far apart mate pairs can be to still be considered for bedtools merging
    downstream.
    """

    ## pipe stats output to grep
    cmd1 = [ipyrad.bins.samtools, "stats", sample.files.mapped_reads]
    cmd2 = ["grep", "SN"]
    proc1 = sps.Popen(cmd1, stderr=sps.STDOUT, stdout=sps.PIPE)
    proc2 = sps.Popen(cmd2, stderr=sps.STDOUT, stdout=sps.PIPE, stdin=proc1.stdout)

    ## get piped result
    res = proc2.communicate()[0]

    ## raise exception on failure and do cleanup
    if proc2.returncode:
        raise IPyradWarningExit("error in %s: %s", cmd2, res)

    ## starting vals
    avg_insert = 0
    stdv_insert = 0
    avg_len = 0

    ## iterate over results
    for line in res.split("\n"):
        if "insert size average" in line:
            avg_insert = float(line.split(":")[-1].strip())

        elif "insert size standard deviation" in line:
            ## hack to fix sim data when stdv is 0.0. Shouldn't
            ## impact real data bcz stdv gets rounded up below
            stdv_insert = float(line.split(":")[-1].strip()) + 0.1

        elif "average length" in line:
            avg_len = float(line.split(":")[-1].strip())

    LOGGER.debug("avg {} stdv {} avg_len {}"\
                 .format(avg_insert, stdv_insert, avg_len))

    ## If all values return successfully set the max inner mate distance.
    ## This is tricky. avg_insert is the average length of R1+R2+inner mate
    ## distance. avg_len is the average length of a read. If there are lots
    ## of reads that overlap then avg_insert will be close to but bigger than
    ## avg_len. We are looking for the right value for `bedtools merge -d`
    ## which wants to know the max distance between reads.
    if all([avg_insert, stdv_insert, avg_len]):
        ## If 2 * the average length of a read is less than the average
        ## insert size then most reads DO NOT overlap
        if stdv_insert < 5:
            stdv_insert = 5.
        if (2 * avg_len) < avg_insert:
            hack = avg_insert + (3 * np.math.ceil(stdv_insert)) - (2 * avg_len)

        ## If it is > than the average insert size then most reads DO
        ## overlap, so we have to calculate inner mate distance a little
        ## differently.
        else:
            hack = (avg_insert - avg_len) + (3 * np.math.ceil(stdv_insert))

        ## set the hackerdict value
        LOGGER.info("stdv: hacked insert size is %s", hack)
        data._hackersonly["max_inner_mate_distance"] = int(np.math.ceil(hack))

    else:
        ## If something fsck then set a relatively conservative distance
        data._hackersonly["max_inner_mate_distance"] = 300
        LOGGER.debug("inner mate distance for {} - {}".format(sample.name,\
                    data._hackersonly["max_inner_mate_distance"]))


Instruction:
Implement a function in Python 3 to
get all contiguous genomic regions with one or more overlapping reads. This is the shell command we'll eventually run bedtools bamtobed -i 1A_0.sorted.bam | bedtools merge [-d 100] -i <input_bam> : specifies the input file to bed'ize -d <int> : For PE set max distance between reads

Response:
def bedtools_merge(data, sample):
    """
    Get all contiguous genomic regions with one or more overlapping
    reads. This is the shell command we'll eventually run

    bedtools bamtobed -i 1A_0.sorted.bam | bedtools merge [-d 100]
        -i <input_bam>  :   specifies the input file to bed'ize
        -d <int>        :   For PE set max distance between reads
    """
    LOGGER.info("Entering bedtools_merge: %s", sample.name)
    mappedreads = os.path.join(data.dirs.refmapping,
                               sample.name+"-mapped-sorted.bam")

    ## command to call `bedtools bamtobed`, and pipe output to stdout
    ## Usage:   bedtools bamtobed [OPTIONS] -i <bam>
    ## Usage:   bedtools merge [OPTIONS] -i <bam>
    cmd1 = [ipyrad.bins.bedtools, "bamtobed", "-i", mappedreads]
    cmd2 = [ipyrad.bins.bedtools, "merge", "-i", "-"]

    ## If PE the -d flag to tell bedtools how far apart to allow mate pairs.
    ## If SE the -d flag is negative, specifying that SE reads need to
    ## overlap by at least a specific number of bp. This prevents the
    ## stairstep syndrome when a + and - read are both extending from
    ## the same cutsite. Passing a negative number to `merge -d` gets this done.
    if 'pair' in data.paramsdict["datatype"]:
        check_insert_size(data, sample)
        #cmd2.insert(2, str(data._hackersonly["max_inner_mate_distance"]))
        cmd2.insert(2, str(data._hackersonly["max_inner_mate_distance"]))
        cmd2.insert(2, "-d")
    else:
        cmd2.insert(2, str(-1 * data._hackersonly["min_SE_refmap_overlap"]))
        cmd2.insert(2, "-d")

    ## pipe output from bamtobed into merge
    LOGGER.info("stdv: bedtools merge cmds: %s %s", cmd1, cmd2)
    proc1 = sps.Popen(cmd1, stderr=sps.STDOUT, stdout=sps.PIPE)
    proc2 = sps.Popen(cmd2, stderr=sps.STDOUT, stdout=sps.PIPE, stdin=proc1.stdout)
    result = proc2.communicate()[0]
    proc1.stdout.close()

    ## check for errors and do cleanup
    if proc2.returncode:
        raise IPyradWarningExit("error in %s: %s", cmd2, result)

    ## Write the bedfile out, because it's useful sometimes.
    if os.path.exists(ipyrad.__debugflag__):
        with open(os.path.join(data.dirs.refmapping, sample.name + ".bed"), 'w') as outfile:
            outfile.write(result)

    ## Report the number of regions we're returning
    nregions = len(result.strip().split("\n"))
    LOGGER.info("bedtools_merge: Got # regions: %s", nregions)
    return result


Instruction:
Can you generate the documentation for the following Python 3 function
def trim_reference_sequence(fasta):
    LOGGER.debug("pre - {}".format(fasta[0]))

    ## If the reads are merged then the reference sequence should be the
    ## same length as the merged pair. If unmerged then we have to fix it.
    if "nnnn" in fasta[1]:
        r1_len = len(fasta[1].split("\n")[1].split("nnnn")[0])
        r2_len = len(fasta[1].split("\n")[1].split("nnnn")[1])
        new_seq = fasta[0].split("\n")[1][:r1_len]+("nnnn")\
                    + revcomp(fasta[0].split("\n")[1][-r2_len:])
        fasta[0] = fasta[0].split("\n")[0]+"\n"+new_seq

    LOGGER.debug("post - {}".format(fasta[0]))
    return fasta

Response: This function trims the reference sequence to the length of the merged sequence.


Instruction:
Can you tell what is the following Python 3 function doing
def bam_region_to_fasta(data, sample, proc1, chrom, region_start, region_end):

    ## output bam file handle for storing genome regions
    bamf = sample.files.mapped_reads
    if not os.path.exists(bamf):
        raise IPyradWarningExit("  file not found - %s", bamf)

    # chrom = re.escape(repr(chrom))[1:-1].replace('\\\\', '\\')
    #LOGGER.info("before: %s", chrom)
    chrom.replace("|", r"\|")
    chrom.replace("(", r"\(")
    chrom.replace(")", r"\)")
    #LOGGER.info("after: %s", chrom)

    ## What we want to do is have the num-chrom dict as an arg, then build this
    ## string as three ints [chrom-int, pos-start, pos-end]
    #cint = cdict[chrom]
    #cpstring = "__{}_{}_{}__".format(cint, int(region_start)+1, region_end)

    ## a string argument as input to commands, indexed at either 0 or 1,
    ## and with pipe characters removed from chromo names
    ## rstring_id1 is for fetching the reference sequence bcz faidx is
    ## 1 indexed
    rstring_id1 = "{}:{}-{}"\
        .format(chrom, str(int(region_start)+1), region_end)

    ## rstring_id0 is just for printing out the reference CHROM/POS
    ## in the read name
    rstring_id0 = "{}:{}-{}"\
        .format(chrom, region_start, region_end)

    ## If SE then we enforce the minimum overlap distance to avoid the
    ## staircase syndrome of multiple reads overlapping just a little.
    overlap_buffer = 0
    if not "pair" in data.paramsdict["datatype"]:
        overlap_buffer = data._hackersonly["min_SE_refmap_overlap"]

    ## rstring_id0_buffered is for samtools view. We have to play patty
    ## cake here with the two rstring_id0s because we want `view` to
    ## enforce the buffer for SE, but we want the reference sequence
    ## start and end positions to print correctly for downstream.
    rstring_id0_buffered = "{}:{}-{}"\
        .format(chrom, int(region_start) + overlap_buffer,\
                int(region_end) - overlap_buffer)

    ## The "samtools faidx" command will grab this region from reference
    ## which we'll paste in at the top of each stack to aid alignment.
    cmd1 = [ipyrad.bins.samtools, "faidx",
            data.paramsdict["reference_sequence"],
            rstring_id1, " ; echo __done__"]

    ## Call the command, I found that it doesn't work with shell=False if
    ## the refstring is 'MT':100-200', but it works if it is MT:100-200.
    LOGGER.info("Grabbing bam_region_to_fasta:\n {}".format(cmd1))
    #proc1 = sps.Popen(cmd1, stderr=sps.STDOUT, stdout=sps.PIPE)
    #ref = proc1.communicate()[0]
    #if proc1.returncode:
    #    raise IPyradWarningExit("  error in %s: %s", cmd1, ref)

    ## push the samtools faidx command to our subprocess, then accumulate
    ## the results from stdout
    print(" ".join(cmd1), file=proc1.stdin)
    ref = ""
    for line in iter(proc1.stdout.readline, "//\n"):
        if "__done__" in line:
            break
        ref += line

    ## initialize the fasta list.
    fasta = []

    ## parse sam to fasta. Save ref location to name.
    ## Set size= an improbably large value so the REF sequence
    ## sorts to the top for muscle aligning.
    try:
        name, seq = ref.strip().split("\n", 1)
        seq = "".join(seq.split("\n"))
        fasta = ["{}_REF;size={};+\n{}".format(name, 1000000, seq)]
    except ValueError as inst:
        LOGGER.error("ref failed to parse - {}".format(ref))
        LOGGER.error(" ".join(cmd1))

    ## if PE then you have to merge the reads here
    if "pair" in data.paramsdict["datatype"]:
        ## PE R1 can either be on the forward or the reverse strand.
        ## Samtools view always outputs reads with respect to the
        ## forward strand. This means that reads with R1 on reverse
        ## end up with the R1 and R2 reference sequences swapped
        ## in the clust.gz file. There is a way to fix it but it's
        ## very annoying and i'm not sure if it's worth it...
        ## Drop the reference sequence for now...
        ##
        ## If you ever fix this be sure to remove the reference sequence
        ## from each cluster post alignment in cluster_within/align_and_parse()
        fasta = []

        ## Create temporary files for R1, R2 and merged, which we will pass to
        ## the function merge_pairs() which calls vsearch to test merging.
        ##
        ## If you are on linux then creating the temp files in /dev/shm
        ## should improve performance
        if os.path.exists("/dev/shm"):
            prefix = os.path.join("/dev/shm",
                            "{}-{}".format(sample.name, rstring_id0))
        else:
            prefix = os.path.join(data.dirs.refmapping,
                            "{}-{}".format(sample.name, rstring_id0))
        read1 = "{}-R1".format(prefix)
        read2 = "{}-R2".format(prefix)
        merged = "{}-merged".format(prefix)

        ## Grab all the reads that map to this genomic location and dump
        ## fastq to R1 and R2 files.
        ## `-v 45` sets the default qscore to something high
        cmd1 = " ".join([ipyrad.bins.samtools, "view", "-b", bamf, rstring_id0])
        cmd2 = " ".join([ipyrad.bins.samtools, "bam2fq", "-v", "45", "-1", read1, "-2", read2, "-", "; echo __done__"])
        cmd = " | ".join([cmd1, cmd2])

        print(cmd, file=proc1.stdin)
        for line in iter(proc1.stdout.readline, "//\n"):
            if "__done__" in line:
                break

        ## run commands, pipe 1 -> 2, then cleanup
        ## proc1 = sps.Popen(cmd1, stderr=sps.STDOUT, stdout=sps.PIPE)
        ## proc2 = sps.Popen(cmd2, stderr=sps.STDOUT, stdout=sps.PIPE, stdin=proc1.stdout)
        ## res = proc2.communicate()[0]
        ## if proc2.returncode:
        ##     raise IPyradWarningExit("error {}: {}".format(cmd2, res))
        ## proc1.stdout.close()

        ## merge the pairs. 0 means don't revcomp bcz samtools already
        ## did it for us. 1 means "actually merge".
        try:
            ## return number of merged reads, writes merged data to 'merged'
            ## we don't yet do anything with the returned number of merged
            _ = merge_pairs(data, [(read1, read2)], merged, 0, 1)

            with open(merged, 'r') as infile:
                quatro = itertools.izip(*[iter(infile)]*4)
                while 1:
                    try:
                        bits = quatro.next()
                    except StopIteration:
                        break
                    ## TODO: figure out a real way to get orientation for PE
                    orient = "+"
                    fullfast = ">{a};{b};{c};{d}\n{e}".format(
                        a=bits[0].split(";")[0],
                        b=rstring_id1,
                        c=bits[0].split(";")[1],
                        d=orient,
                        e=bits[1].strip())
                    #,e=bits[9])
                    fasta.append(fullfast)

                ## TODO: If you ever figure out a good way to get the reference
                ## sequence included w/ PE then this commented call is useful
                ## for trimming the reference sequence to be the right length.
                ## If doing PE and R1/R2 don't overlap then the reference sequence
                ## will be quite long and will cause indel hell during the
                ## alignment stage. Here trim the reference sequence to the length
                ## of the merged reads.
                ## This is commented out because we aren't currently including the
                ## ref seq for PE alignment.
                #fasta = trim_reference_sequence(fasta)

        except (OSError, ValueError, IPyradError) as inst:
            ## ValueError raised inside merge_pairs() if it can't open one
            ## or both of the files. Write this out, but ignore for now.
            ## Failed merging, probably unequal number of reads in R1 and R2
            ## IPyradError raised if merge_pairs can't read either R1 or R2
            ## file.
            ## Skip this locus?
            LOGGER.debug("Failed to merge reads, continuing; %s", inst)
            LOGGER.error("cmd - {}".format(cmd))
            return ""
        finally:
            ## Only clean up the files if they exist otherwise it'll raise.
            if os.path.exists(merged):
                os.remove(merged)
            if os.path.exists(read1):
                os.remove(read1)
            if os.path.exists(read2):
                os.remove(read2)

    else:
        try:
            ## SE if faster than PE cuz it skips writing intermedidate files
            ## rstring_id0_buffered is going to enforce the required
            ## min_SE_refmap_overlap on either end of this region.
            cmd2 = [ipyrad.bins.samtools, "view", bamf, rstring_id0_buffered]
            proc2 = sps.Popen(cmd2, stderr=sps.STDOUT, stdout=sps.PIPE)

            ## run and check outputs
            res = proc2.communicate()[0]
            if proc2.returncode:
                raise IPyradWarningExit("{} {}".format(cmd2, res))

            ## if the region string is malformated you'll get back a warning
            ## from samtools
            if "[main_samview]" in res:
                raise IPyradError("Bad reference region {}".format(rstring_id0_buffered))

            ## do not join seqs that
            for line in res.strip().split("\n"):
                bits = line.split("\t")

                ## Read in the 2nd field (FLAGS), convert to binary
                ## and test if the 7th bit is set which indicates revcomp
                orient = "+"
                if int('{0:012b}'.format(int(bits[1]))[7]):
                    orient = "-"
                    ## Don't actually revcomp the sequence because samtools
                    ## writes out reference sequence on the forward strand
                    ## as well as reverse strand hits from the bam file.
                    #bits[9] = revcomp(bits[9])

                ## Rip insert the mapping position between the seq label and
                ## the vsearch derep size.
                fullfast = ">{a};{b};{c};{d}\n{e}".format(
                    a=bits[0].split(";")[0],
                    b=rstring_id0,
                    c=bits[0].split(";")[1],
                    d=orient,
                    e=bits[9])
                fasta.append(fullfast)
        except IPyradError as inst:
            ## If the mapped fragment is too short then the you'll get
            ## regions that look like this: scaffold262:299039-299036
            ## Just carry on, it's not a big deal.
            LOGGER.debug("Got a bad region string: {}".format(inst))
            return ""
        except (OSError, ValueError, Exception) as inst:
            ## Once in a blue moon something fsck and it breaks the
            ## assembly. No reason to give up if .001% of reads fail
            ## so just skip this locus.
            LOGGER.error("Failed get reads at a locus, continuing; %s", inst)
            LOGGER.error("cmd - {}".format(cmd2))
            return ""

    return "\n".join(fasta)

Response: This function takes the chromosome position start and end bases and returns the sequence of all reads that overlap these sites and reads that overlap these sites.


Instruction:
Given the following Python 3 function, write the documentation
def refmap_stats(data, sample):
    ## shorter names
    mapf = os.path.join(data.dirs.refmapping, sample.name+"-mapped-sorted.bam")
    umapf = os.path.join(data.dirs.refmapping, sample.name+"-unmapped.bam")

    ## get from unmapped
    cmd1 = [ipyrad.bins.samtools, "flagstat", umapf]
    proc1 = sps.Popen(cmd1, stderr=sps.STDOUT, stdout=sps.PIPE)
    result1 = proc1.communicate()[0]

    ## get from mapped
    cmd2 = [ipyrad.bins.samtools, "flagstat", mapf]
    proc2 = sps.Popen(cmd2, stderr=sps.STDOUT, stdout=sps.PIPE)
    result2 = proc2.communicate()[0]

    ## store results
    ## If PE, samtools reports the _actual_ number of reads mapped, both
    ## R1 and R2, so here if PE divide the results by 2 to stay consistent
    ## with how we've been reporting R1 and R2 as one "read pair"
    if "pair" in data.paramsdict["datatype"]:
        sample.stats["refseq_unmapped_reads"] = int(result1.split()[0]) / 2
        sample.stats["refseq_mapped_reads"] = int(result2.split()[0]) / 2
    else:
        sample.stats["refseq_unmapped_reads"] = int(result1.split()[0])
        sample.stats["refseq_mapped_reads"] = int(result2.split()[0])

    sample_cleanup(data, sample)

Response: Get the number of mapped and unmapped reads for a sample and update sample. stats


Instruction:
Can you implement a function in Python 3 that
initializes the refmap files for the given sample

Response:
def refmap_init(data, sample, force):
    """ create some file handles for refmapping """
    ## make some persistent file handles for the refmap reads files
    sample.files.unmapped_reads = os.path.join(data.dirs.edits,
                                  "{}-refmap_derep.fastq".format(sample.name))
    sample.files.mapped_reads = os.path.join(data.dirs.refmapping,
                                  "{}-mapped-sorted.bam".format(sample.name))


Instruction:
How would you explain what the following Python 3 function does
def parse_command_line():

    ## create the parser
    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
  * Example command-line usage ----------------------------------------------

  * Read in sequence/SNP data file, provide linkage, output name, ambig option.
     tetrad -s data.snps.phy -n test             ## input phylip and give name
     tetrad -s data.snps.phy -l data.snps.map    ## sample one SNP per locus
     tetrad -s data.snps.phy -n noambigs -r 0    ## do not use hetero sites

  * Load saved/checkpointed analysis from '.tet.json' file, or force restart.
     tetrad -j test.tet.json -b 100         ## continue 'test' until 100 boots
     tetrad -j test.tet.json -b 100 -f      ## force restart of 'test'

  * Sampling modes: 'equal' uses guide tree to sample quartets more efficiently
     tetrad -s data.snps.phy -m all                       ## sample all quartets
     tetrad -s data.snps.phy -m random -q 1e6 -x 123      ## sample 1M randomly
     tetrad -s data.snps.phy -m equal -q 1e6 -t guide.tre ## sample 1M across tree

  * Connect to N cores on a computer (default without -c arg is to use all avail.)
     tetrad -s data.snps.phy -c 20

  * Start an MPI cluster to connect to nodes across multiple available hosts.
     tetrad -s data.snps.phy --MPI

  * Connect to a manually started ipcluster instance with default or named profile
     tetrad -s data.snps.phy --ipcluster        ## connects to default profile
     tetrad -s data.snps.phy --ipcluster pname  ## connects to profile='pname'

  * Further documentation: http://ipyrad.readthedocs.io/analysis.html
    """)

    ## get version from ipyrad
    ipyversion = str(pkg_resources.get_distribution('ipyrad'))
    parser.add_argument('-v', '--version', action='version',
        version="tetrad "+ipyversion.split()[1])

    parser.add_argument('-f', "--force", action='store_true',
        help="force overwrite of existing data")

    parser.add_argument('-s', metavar="seq", dest="seq",
        type=str, default=None,
        help="path to input phylip file (only SNPs)")

    parser.add_argument('-j', metavar='json', dest="json",
        type=str, default=None,
        help="load checkpointed/saved analysis from JSON file.")

    parser.add_argument('-m', metavar="method", dest="method",
        type=str, default="all",
        help="method for sampling quartets (all, random, or equal)")

    parser.add_argument('-q', metavar="nquartets", dest="nquartets",
        type=int, default=0,
        help="number of quartets to sample (if not -m all)")

    parser.add_argument('-b', metavar="boots", dest="boots",
        type=int, default=0,
        help="number of non-parametric bootstrap replicates")

    parser.add_argument('-l', metavar="map_file", dest="map",
        type=str, default=None,
        help="map file of snp linkages (e.g., ipyrad .snps.map)")

    parser.add_argument('-r', metavar="resolve", dest='resolve',
        type=int, default=1,
        help="randomly resolve heterozygous sites (default=1)")

    parser.add_argument('-n', metavar="name", dest="name",
        type=str, default="test",
        help="output name prefix (default: 'test')")

    parser.add_argument('-o', metavar="workdir", dest="workdir",
        type=str, default="./analysis-tetrad",
        help="output directory (default: creates ./analysis-tetrad)")

    parser.add_argument('-t', metavar="starting_tree", dest="tree",
        type=str, default=None,
        help="newick file starting tree for equal splits sampling")

    parser.add_argument("-c", metavar="CPUs/cores", dest="cores",
        type=int, default=0,
        help="setting -c improves parallel efficiency with --MPI")

    parser.add_argument("-x", metavar="random_seed", dest="rseed",
        type=int, default=None,
        help="random seed for quartet sampling and/or bootstrapping")

    parser.add_argument('-d', "--debug", action='store_true',
        help="print lots more info to debugger: ipyrad_log.txt.")

    parser.add_argument("--MPI", action='store_true',
        help="connect to parallel CPUs across multiple nodes")

    parser.add_argument("--invariants", action='store_true',
        help="save a (large) database of all invariants")

    parser.add_argument("--ipcluster", metavar="ipcluster", dest="ipcluster",
        type=str, nargs="?", const="default",
        help="connect to ipcluster profile (default: 'default')")

    ## if no args then return help message
    if len(sys.argv) == 1:
        parser.print_help()
        sys.exit(1)

    ## parse args
    args = parser.parse_args()

    ## RAISE errors right away for some bad argument combinations:
    if args.method not in ["random", "equal", "all"]:
        raise IPyradWarningExit("  method argument (-m) must be one of"+\
        """ "all", "random", or "equal.\n""")

    ## if 'random' require nquarts argument
    #if args.method == 'random':
    #    if not args.nquartets:
    #        raise IPyradWarningExit(\
    #        "  Number of quartets (-q) is required with method = random\n")

    ## if 'equal' method require starting tree and nquarts
    # if args.method == 'equal':
    #     raise IPyradWarningExit(\
    #         "  The equal sampling method is currently for developers only.\n")
    #     if not args.nquartets:
    #         raise IPyradWarningExit(\
    #         "  Number of quartets (-q) is required with method = equal\n")
    #     if not args.tree:
    #         raise IPyradWarningExit(\
    #         "  Input guide tree (-t) is required with method = equal\n")

    ## required args
    if not any(x in ["seq", "json"] for x in vars(args).keys()):
        print("""
    Bad arguments: tetrad command must include at least one of (-s or -j)
    """)
        parser.print_help()
        sys.exit(1)

    return args

Response:
Parse command - line arguments for the
 formula.


Instruction:
Can you tell what is the following Python 3 function doing
def main():

    ## parse params file input (returns to stdout if --help or --version)
    args = parse_command_line()
    print(HEADER.format(ip.__version__))

    ## set random seed
    np.random.seed(args.rseed)

    ## debugger----------------------------------------
    if os.path.exists(ip.__debugflag__):
        os.remove(ip.__debugflag__)
    if args.debug:
        print("\n  ** Enabling debug mode ** ")
        ip._debug_on()

    ## if JSON, load existing Tetrad analysis -----------------------
    if args.json:
        data = ipa.tetrad(name=args.name, workdir=args.workdir, load=True)
        ## if force then remove all results
        if args.force:
            data._refresh()

    ## else create a new tmp assembly for the seqarray-----------------
    else:
        ## create new Tetrad class Object if it doesn't exist
        newjson = os.path.join(args.workdir, args.name+'.tet.json')
        ## if not quiet...
        print("tetrad instance: {}".format(args.name))

        if (not os.path.exists(newjson)) or args.force:
            ## purge any files associated with this name if forced
            if args.force:
                ## init an object in the correct location just to refresh
                ipa.tetrad(name=args.name,
                           workdir=args.workdir,
                           data=args.seq,
                           initarr=False,
                           save_invariants=args.invariants,
                           cli=True,
                           quiet=True)._refresh()

            ## create new tetrad object
            data = ipa.tetrad(name=args.name,
                              workdir=args.workdir,
                              method=args.method,
                              data=args.seq,
                              resolve=args.resolve,
                              mapfile=args.map,
                              guidetree=args.tree,
                              nboots=args.boots,
                              nquartets=args.nquartets,
                              cli=True,
                              save_invariants=args.invariants,
                              )
        else:
            raise SystemExit(QUARTET_EXISTS\
            .format(args.name, args.workdir, args.workdir, args.name, args.name))

    ## boots can be set either for a new object or loaded JSON to continue it
    if args.boots:
        data.params.nboots = int(args.boots)

    ## if ipyclient is running (and matched profile) then use that one
    if args.ipcluster:
        ipyclient = ipp.Client(profile=args.ipcluster)
        data._ipcluster["cores"] = len(ipyclient)

    ## if not then we need to register and launch an ipcluster instance
    else:
        ## set CLI ipcluster terms
        ipyclient = None
        data._ipcluster["cores"] = args.cores if args.cores else detect_cpus()
        data._ipcluster["engines"] = "Local"
        if args.MPI:
            data._ipcluster["engines"] = "MPI"
            if not args.cores:
                raise IPyradWarningExit("must provide -c argument with --MPI")
        ## register to have a cluster-id with "ip- name"
        data = register_ipcluster(data)

    ## message about whether we are continuing from existing
    if data.checkpoint.boots:
        print(LOADING_MESSAGE.format(
            data.name, data.params.method, data.checkpoint.boots))

    ## run tetrad main function within a wrapper. The wrapper creates an
    ## ipyclient view and appends to the list of arguments to run 'run'.
    data.run(force=args.force, ipyclient=ipyclient)

Response: main function for the sequence array analysis


Instruction:
Given the following Python 3 function, write the documentation
def _command_list(self):

        ## base args
        cmd = [self.params.binary,
                "-i", OPJ(self.workdir, self.name+".treemix.in.gz"),
                "-o", OPJ(self.workdir, self.name),
                ]

        ## addon params
        args = []
        for key, val in self.params:
            if key not in ["minmap", "binary"]:
                if key == "g":
                    if val[0]:
                        args += ["-"+key, str(val[0]), str(val[1])]
                elif key == "global_":
                    if val:
                        args += ["-"+key[:-1]]
                elif key in ["se", "global", "noss"]:
                    if val:
                        args += ["-"+key]
                else:
                    if val:
                        args += ["-"+key, str(val)]

        return cmd+args

Response: build the command list


Instruction:
How would you code a function in Python 3 to
return a subsample of unlinked snp sites

Response:
def _subsample(self):
        """ returns a subsample of unlinked snp sites """
        spans = self.maparr
        samp = np.zeros(spans.shape[0], dtype=np.uint64)
        for i in xrange(spans.shape[0]):
            samp[i] = np.random.randint(spans[i, 0], spans[i, 1], 1)
        return samp


Instruction:
How would you explain what the following Python 3 function does
def copy(self, name):

        ## make deepcopy of self.__dict__ but do not copy async objects
        subdict = {i:j for i, j in self.__dict__.iteritems() if i != "asyncs"}
        newdict = copy.deepcopy(subdict)

        ## make back into a Treemix object
        #if name == self.name:
        #    raise Exception("new object name must be different from its parent")

        newobj = Treemix(
            data=newdict["data"],
            name=name,
            workdir=newdict["workdir"],
            imap={i:j for i, j in newdict["imap"].items()},
            mapfile=newdict['mapfile'],
            minmap={i:j for i, j in newdict["minmap"].items()},
            seed=np.random.randint(0, int(1e9)),
            )

        ## update special dict attributes but not files
        for key, val in newobj.params.__dict__.iteritems():
            newobj.params.__setattr__(key, self.params.__getattribute__(key))
        #for key, val in newobj.filters.__dict__.iteritems():
        #    newobj.filters.__setattr__(key, self.filters.__getattribute__(key))

        ## new object must have a different name than it's parent
        return newobj

Response:
Returns a copy of the treemix object with the same parameter settings
        but with the files attributes cleared and a new name attribute.


Instruction:
How would you code a function in Python 3 to
return a toyplot. axes object.

Response:
def draw(self, axes):
        """
        Returns a treemix plot on a toyplot.axes object.
        """
        ## create a toytree object from the treemix tree result
        tre = toytree.tree(newick=self.results.tree)
        tre.draw(
            axes=axes,
            use_edge_lengths=True,
            tree_style='c',
            tip_labels_align=True,
            edge_align_style={"stroke-width": 1}
        );

        ## get coords
        for admix in self.results.admixture:
            ## parse admix event
            pidx, pdist, cidx, cdist, weight = admix
            a = _get_admix_point(tre, pidx, pdist)
            b = _get_admix_point(tre, cidx, cdist)

            ## add line for admixture edge
            mark = axes.plot(
                a = (a[0], b[0]),
                b = (a[1], b[1]),
                style={"stroke-width": 10*weight,
                       "stroke-opacity": 0.95,
                       "stroke-linecap": "round"}
            )

            ## add points at admixture sink
            axes.scatterplot(
                a = (b[0]),
                b = (b[1]),
                size=8,
                title="weight: {}".format(weight),
            )

        ## add scale bar for edge lengths
        axes.y.show=False
        axes.x.ticks.show=True
        axes.x.label.text = "Drift parameter"
        return axes


Instruction:
Given the following Python 3 function, write the documentation
def _resolveambig(subseq):
    N = []
    for col in subseq:
        rand = np.random.binomial(1, 0.5)
        N.append([_AMBIGS[i][rand] for i in col])
    return np.array(N)

Response: Randomly resolves iupac hetero codes.


Instruction:
Write a Python 3 script to
count the number of PIS in a loci sequence

Response:
def _count_PIS(seqsamp, N):
    """ filters for loci with >= N PIS """
    counts = [Counter(col) for col in seqsamp.T if not ("-" in col or "N" in col)]
    pis = [i.most_common(2)[1][1] > 1 for i in counts if len(i.most_common(2))>1]
    if sum(pis) >= N:
        return sum(pis)
    else:
        return 0


Instruction:
Can you tell what is the following Python 3 function doing
def write_nexus_files(self, force=False, quiet=False):

        ## clear existing files
        existing = glob.glob(os.path.join(self.workdir, self.name, "*.nex"))
        if any(existing):
            if force:
                for rfile in existing:
                    os.remove(rfile)
            else:
                path = os.path.join(self.workdir, self.name)
                raise IPyradWarningExit(EXISTING_NEX_FILES.format(path))

        ## parse the loci or alleles file
        with open(self.files.data) as infile:
            loci = iter(infile.read().strip().split("|\n"))

        ## use entered samples or parse them from the file
        if not self.samples:
            with open(self.files.data) as infile:
                samples = set((i.split()[0] for i in infile.readlines() \
                               if "//" not in i))
        else:
            samples = set(self.samples)

        ## keep track of how many loci pass filtering
        totn = len(samples)
        nloci = 0

        ## this set is just used for matching, then we randomly
        ## subsample for real within the locus so it varies
        if self._alleles:
            msamples = {i+rbin() for i in samples}
        else:
            msamples = samples

        ## write subsampled set of loci
        for loc in loci:
            ## get names and seqs from locus
            dat = loc.split("\n")[:-1]
            try:
                names = [i.split()[0] for i in dat]
                snames = set(names)
                seqs = np.array([list(i.split()[1]) for i in dat])
            except IndexError:
                print(ALLELESBUGFIXED)
                continue

            ## check name matches
            if len(snames.intersection(msamples)) == totn:

                ## prune sample names if alleles. Done here so it is randomly
                ## different in every locus which allele is selected from
                ## each sample (e.g., 0 or 1)
                if self._alleles:
                    _samples = [i+rbin() for i in samples]
                else:
                    _samples = samples

                ## re-order seqs to be in set order
                seqsamp = seqs[[names.index(tax) for tax in _samples]]

                ## resolve ambiguities randomly if .loci file otherwise
                ## sample one of the alleles if .alleles file.
                if not self._alleles:
                    seqsamp = _resolveambig(seqsamp)

                ## find parsimony informative sites
                if _count_PIS(seqsamp, self.params.minsnps):
                    ## keep the locus
                    nloci += 1

                    ## remove empty columns given this sampling
                    copied = seqsamp.copy()
                    copied[copied == "-"] == "N"
                    rmcol = np.all(copied == "N", axis=0)
                    seqsamp = seqsamp[:, ~rmcol]

                    ## write nexus file
                    if self._alleles:
                        ## trim off the allele number
                        samps = [i.rsplit("_", 1)[0] for i in _samples]
                        mdict = dict(zip(samps, [i.tostring() for i in seqsamp]))
                    else:
                        mdict = dict(zip(_samples, [i.tostring() for i in seqsamp]))
                    self._write_nex(mdict, nloci)

                    ## quit early if using maxloci
                    if nloci == self.params.maxloci:
                        break

        ## print data size
        if not quiet:
            path = os.path.join(self.workdir, self.name)
            path = path.replace(os.path.expanduser("~"), "~")
            print("wrote {} nexus files to {}".format(nloci, path))

Response: Write nexus files to the directory.


Instruction:
Write a Python 3 script for
running the nexus bucky analysis on the given set of parameters.

Response:
def run(self, steps=None, ipyclient=None, force=False, quiet=False):
        """
        Submits an ordered list of jobs to a load-balancer to complete
        the following tasks, and reports a progress bar:
        (1) Write nexus files for each locus
        (2) Run mrBayes on each locus to get a posterior of gene trees
        (3) Run mbsum (a bucky tool) on the posterior set of trees
        (4) Run Bucky on the summarized set of trees for all alpha values.

        Parameters:
        -----------
        ipyclient (ipyparallel.Client())
            A connected ipyparallel Client object used to distribute jobs
        force (bool):
            Whether to overwrite existing files with the same name and workdir
            if they exist. Default is False.
        quiet (bool):
            Whether to suppress progress information. Default is False.
        steps (list):
            A list of integers of steps to perform. This is useful if a
            job was interrupted, or you created a new bucky object copy,
            or you wish to run an analysis under a new set of parameters,
            after having run it once. For example, if you finished running
            steps 1 and 2 (write nexus files and infer mrbayes posteriors),
            but you want to rerun steps 3 and 4 with new settings, then you
            could enter `steps=[3,4]` and also `force=True` to run steps 3
            and 4 with a new set of parameters. Default argument is None
            which means run all steps.
        """

        ## require ipyclient
        if not ipyclient:
            raise IPyradWarningExit("an ipyclient object is required")

        ## check the steps argument
        if not steps:
            steps = [1, 2, 3, 4]
        if isinstance(steps, (int, str)):
            steps = [int(i) for i in [steps]]
        if isinstance(steps, list):
            if not all(isinstance(i, int) for i in steps):
                raise IPyradWarningExit("steps must be a list of integers")

        ## run steps ------------------------------------------------------
        ## TODO: wrap this function so it plays nice when interrupted.
        if 1 in steps:
            self.write_nexus_files(force=force, quiet=quiet)
        if 2 in steps:
            self.run_mrbayes(force=force, quiet=quiet, ipyclient=ipyclient)
        if 3 in steps:
            self.run_mbsum(force=force, quiet=quiet, ipyclient=ipyclient)
        if 4 in steps:
            self.run_bucky(force=force, quiet=quiet, ipyclient=ipyclient)

        ## make sure jobs are done if waiting (TODO: maybe make this optional)
        ipyclient.wait()


Instruction:
How would you code a function in Python 3 to
write a NEX file with a mrbayes analysis block given a set of mcmc arguments.

Response:
def _write_nex(self, mdict, nlocus):
        """
        function that takes a dictionary mapping names to sequences,
        and a locus number, and writes it as a NEXUS file with a mrbayes
        analysis block given a set of mcmc arguments.
        """

        ## create matrix as a string
        max_name_len = max([len(i) for i in mdict])
        namestring = "{:<" + str(max_name_len+1) + "} {}\n"
        matrix = ""
        for i in mdict.items():
            matrix += namestring.format(i[0], i[1])

        ## ensure dir
        minidir = os.path.realpath(os.path.join(self.workdir, self.name))
        if not os.path.exists(minidir):
            os.makedirs(minidir)

        ## write nexus block
        handle = os.path.join(minidir, "{}.nex".format(nlocus))
        with open(handle, 'w') as outnex:
            outnex.write(NEXBLOCK.format(**{
                "ntax": len(mdict),
                "nchar": len(mdict.values()[0]),
                "matrix": matrix,
                "ngen": self.params.mb_mcmc_ngen,
                "sfreq": self.params.mb_mcmc_sample_freq,
                "burnin": self.params.mb_mcmc_burnin,
                }))


Instruction:
Given the following Python 3 function, write the documentation
def run_mbsum(self, ipyclient, force=False, quiet=False):
        minidir = os.path.realpath(os.path.join(self.workdir, self.name))
        trees1 = glob.glob(os.path.join(minidir, "*.run1.t"))
        trees2 = glob.glob(os.path.join(minidir, "*.run2.t"))

        ## clear existing files
        existing = glob.glob(os.path.join(self.workdir, self.name, "*.sumt"))
        if any(existing):
            if force:
                for rfile in existing:
                    os.remove(rfile)
            else:
                path = os.path.join(self.workdir, self.name)
                raise IPyradWarningExit(EXISTING_SUMT_FILES.format(path))

        ## load balancer
        lbview = ipyclient.load_balanced_view()

        ## submit each to be processed
        asyncs = []
        for tidx in xrange(len(trees1)):
            rep1 = trees1[tidx]
            rep2 = trees2[tidx]
            outname = os.path.join(minidir, str(tidx)+".sumt")
            async = lbview.apply(_call_mbsum, *(rep1, rep2, outname))
            asyncs.append(async)

        ## track progress
        start = time.time()
        printstr = "[mbsum] sum replicate runs      | {} | "
        while 1:
            ready = [i.ready() for i in asyncs]
            elapsed = datetime.timedelta(seconds=int(time.time()-start))
            if not quiet:
                progressbar(len(ready), sum(ready), printstr.format(elapsed), spacer="")
            if len(ready) == sum(ready):
                if not quiet:
                    print("")
                break
            else:
                time.sleep(0.1)

        ## check success
        for async in asyncs:
            if not async.successful():
                raise IPyradWarningExit(async.result())

Response: Sums two replicate mrbayes runs for each locus.


Instruction:
Can you create a Python 3 function that
runs the mrbayes block in each nexus file

Response:
def run_mrbayes(self, ipyclient, force=False, quiet=False):
        """
        calls the mrbayes block in each nexus file.
        """

        ## get all the nexus files for this object
        minidir = os.path.realpath(os.path.join(self.workdir, self.name))
        nexus_files = glob.glob(os.path.join(minidir, "*.nex"))

        ## clear existing files
        #existing = glob.glob(os.path.join(self.workdir, self.name, "*.nex"))
        existing = glob.glob(os.path.join(minidir, "*.nex.*"))
        if any(existing):
            if force:
                for rfile in existing:
                    os.remove(rfile)
            else:
                raise IPyradWarningExit(EXISTING_NEXdot_FILES.format(minidir))

        ## write new nexus files, or should users do that before this?
        #self.write_nexus_files(force=True)

        ## load balancer
        lbview = ipyclient.load_balanced_view()

        ## submit each to be processed
        asyncs = []
        for nex in nexus_files:
            async = lbview.apply(_call_mb, nex)
            asyncs.append(async)

        ## track progress
        start = time.time()
        printstr = "[mb] infer gene-tree posteriors | {} | "
        while 1:
            ready = [i.ready() for i in asyncs]
            elapsed = datetime.timedelta(seconds=int(time.time()-start))
            if not quiet:
                progressbar(len(ready), sum(ready), printstr.format(elapsed), spacer="")
            if len(ready) == sum(ready):
                if not quiet:
                    print("")
                break
            else:
                time.sleep(0.1)

        ## check success
        for async in asyncs:
            if not async.successful():
                raise IPyradWarningExit(async.result())


Instruction:
Can you write a function in Python 3 where it
runs bucky on a set of parameters and stores the result in the ipa. bucky object.

Response:
def run_bucky(self, ipyclient, force=False, quiet=False, subname=False):
        """
        Runs bucky for a given set of parameters and stores the result
        to the ipa.bucky object. The results will be stored by default
        with the name '{name}-{alpha}' unless a argument is passed for
        'subname' to customize the output name.

        Parameters:
        -----------
        subname (str):
            A custom name prefix for the output files produced by the bucky
            analysis and output into the {workdir}/{name} directory.
        force (bool):
            If True then existing result files with the same name prefix
            will be overwritten.
        quiet (bool):
            If True the progress bars will be suppressed.
        ipyclient (ipyparallel.Client)
            An active ipyparallel client to distribute jobs to.

        """

        ## check for existing results files
        minidir = os.path.realpath(os.path.join(self.workdir, self.name))
        infiles = glob.glob(os.path.join(minidir, "*.sumt"))
        outroot = os.path.realpath(os.path.join(self.workdir, self.name))

        ## build alpha list
        if isinstance(self.params.bucky_alpha, list):
            alphas = self.params.bucky_alpha
        else:
            alphas = [self.params.bucky_alpha]

        ## load balancer
        lbview = ipyclient.load_balanced_view()

        ## submit each to be processed
        asyncs = []
        for alpha in alphas:
            pathname = os.path.join(outroot, "CF-a"+str(alpha))
            if (os.path.exists(pathname)) and (force!=True):
                print("BUCKy results already exist for this object at alpha={}\n".format(alpha) +\
                      "use force=True to overwrite existing results")
            else:
                args = [
                    alpha,
                    self.params.bucky_nchains,
                    self.params.bucky_nreps,
                    self.params.bucky_niter,
                    pathname,
                    infiles]
                async = lbview.apply(_call_bucky, *args)
                asyncs.append(async)

        ## track progress
        start = time.time()
        printstr = "[bucky] infer CF posteriors     | {} | "
        while 1:
            ready = [i.ready() for i in asyncs]
            elapsed = datetime.timedelta(seconds=int(time.time()-start))
            if not quiet:
                progressbar(len(ready), sum(ready), printstr.format(elapsed), spacer="")
            if len(ready) == sum(ready):
                if not quiet:
                    print("")
                break
            else:
                time.sleep(0.1)

        ## check success
        for async in asyncs:
            if not async.successful():
                raise IPyradWarningExit(async.result())


Instruction:
Make a summary of the following Python 3 code
def _get_samples(self, samples):
    ## if samples not entered use all samples
    if not samples:
        samples = self.samples.keys()

    ## Be nice and allow user to pass in only one sample as a string,
    ## rather than a one element list. When you make the string into a list
    ## you have to wrap it in square braces or else list makes a list of
    ## each character individually.
    if isinstance(samples, str):
        samples = list([samples])

    ## if sample keys, replace with sample obj
    assert isinstance(samples, list), \
        "to subselect samples enter as a list, e.g., [A, B]."
    newsamples = [self.samples.get(key) for key in samples \
                  if self.samples.get(key)]
    strnewsamples = [i.name for i in newsamples]

    ## are there any samples that did not make it into the dict?
    badsamples = set(samples).difference(set(strnewsamples))
    if badsamples:
        outstring = ", ".join(badsamples)
        raise IPyradError(\
        "Unrecognized Sample name(s) not linked to {}: {}"\
        .format(self.name, outstring))

    ## require Samples
    assert newsamples, \
           "No Samples passed in and none in assembly {}".format(self.name)

    return newsamples

Response:
Internal function to read in perhaps
    non empty list of samples.


Instruction:
Given the following Python 3 function, write the documentation
def _name_from_file(fname, splitnames, fields):
    ## allowed extensions
    file_extensions = [".gz", ".fastq", ".fq", ".fasta", ".clustS", ".consens"]
    base, _ = os.path.splitext(os.path.basename(fname))

    ## remove read number from name
    base = base.replace("_R1_.", ".")\
               .replace("_R1_", "")\
               .replace("_R1.", ".")

    ## remove extensions, retains '.' in file names.
    while 1:
        tmpb, tmpext = os.path.splitext(base)
        if tmpext in file_extensions:
            base = tmpb
        else:
            break

    if fields:
        namebits = base.split(splitnames)
        base = []
        for field in fields:
            try:
                base.append(namebits[field])
            except IndexError:
                pass
        base = splitnames.join(base)

    if not base:
        raise IPyradError("""
    Found invalid/empty filename in link_fastqs. Check splitnames argument.
    """)

    return base

Response: internal function to get the sample name from a pyrad file.


Instruction:
Can you tell what is the following Python 3 function doing
def _read_sample_names(fname):
    try:
        with open(fname, 'r') as infile:
            subsamples = [x.split()[0] for x in infile.readlines() if x.strip()]

    except Exception as inst:
        print("Failed to read input file with sample names.\n{}".format(inst))
        raise inst

    return subsamples

Response: Read in sample names from a plain text file.


Instruction:
Here you have a function in Python 3, explain what it does
def _expander(namepath):
    if "~" in namepath:
        namepath = os.path.expanduser(namepath)
    else:
        namepath = os.path.abspath(namepath)
    return namepath

Response: expand ./ ~ and ../ designators in location names


Instruction:
Can you generate a brief explanation for the following Python 3 code
def merge(name, assemblies):

    ## checks
    assemblies = list(assemblies)

    ## create new Assembly as a branch (deepcopy)
    merged = assemblies[0].branch(name)

    ## get all sample names from all Assemblies
    allsamples = set(merged.samples.keys())
    for iterass in assemblies[1:]:
        allsamples.update(set(iterass.samples.keys()))

    ## Make sure we have the max of all values for max frag length
    ## from all merging assemblies.
    merged._hackersonly["max_fragment_length"] =\
        max([x._hackersonly["max_fragment_length"] for x in assemblies])

    ## warning message?
    warning = 0

    ## iterate over assembly objects, skip first already copied
    for iterass in assemblies[1:]:
        ## iterate over allsamples, add if not in merged
        for sample in iterass.samples:
            ## iterate over stats, skip 'state'
            if sample not in merged.samples:
                merged.samples[sample] = copy.deepcopy(iterass.samples[sample])
                ## if barcodes data present then keep it
                if iterass.barcodes.get(sample):
                    merged.barcodes[sample] = iterass.barcodes[sample]
            else:
                ## merge stats and files of the sample
                for stat in merged.stats.keys()[1:]:
                    merged.samples[sample].stats[stat] += \
                                iterass.samples[sample].stats[stat]
                ## merge file references into a list
                for filetype in ['fastqs', 'edits']:
                    merged.samples[sample].files[filetype] += \
                                iterass.samples[sample].files[filetype]
                if iterass.samples[sample].files["clusters"]:
                    warning += 1

    ## print warning if clusters or later was present in merged assembly
    if warning:
        print("""\
    Warning: the merged Assemblies contained Samples that are identically named,
    and so ipyrad has attempted to merge these Samples. This is perfectly fine to
    do up until step 3, but not after, because at step 3 all reads for a Sample
    should be included during clustering/mapping. Take note, you can merge Assemblies
    at any step *if they do not contain the same Samples*, however, here that is not
    the case. If you wish to proceed with this merged Assembly you will have to
    start from step 3, therefore the 'state' of the Samples in this new merged
    Assembly ({}) have been set to 2.
    """.format(name))
        for sample in merged.samples:
            merged.samples[sample].stats.state = 2
            ## clear stats
            for stat in ["refseq_mapped_reads", "refseq_unmapped_reads",
                         "clusters_total", "clusters_hidepth", "hetero_est",
                         "error_est", "reads_consens"]:
                merged.samples[sample].stats[stat] = 0
            ## clear files
            for ftype in ["mapped_reads", "unmapped_reads", "clusters",
                          "consens", "database"]:
                merged.samples[sample].files[ftype] = []

    ## Set the values for some params that don't make sense inside
    ## merged assemblies
    merged_names = ", ".join([x.name for x in assemblies])
    merged.paramsdict["raw_fastq_path"] = "Merged: " + merged_names
    merged.paramsdict["barcodes_path"] = "Merged: " + merged_names
    merged.paramsdict["sorted_fastq_path"] = "Merged: " + merged_names

    ## return the new Assembly object
    merged.save()
    return merged

Response: Creates and returns a new Assembly object in which samples from two or more Assemblies are merged.


Instruction:
Can you write a function in Python 3 where it
counts the number of lines in a file.

Response:
def _bufcountlines(filename, gzipped):
    """
    fast line counter. Used to quickly sum number of input reads when running
    link_fastqs to append files. """
    if gzipped:
        fin = gzip.open(filename)
    else:
        fin = open(filename)
    nlines = 0
    buf_size = 1024 * 1024
    read_f = fin.read # loop optimization
    buf = read_f(buf_size)
    while buf:
        nlines += buf.count('\n')
        buf = read_f(buf_size)
    fin.close()
    return nlines