Instruction:
Write a Python 3 script for
entering funcs for SE or merged data

Response:
def enter_singles(iloc, pnames, snppad, edg, aseqs, asnps, smask, samplecov, locuscov, start):
    """ enter funcs for SE or merged data """

    ## grab all seqs between edges
    seq = aseqs[iloc, :, edg[0]:edg[1]+1]
    ## snps was created using only the selected samples, and is edge masked.
    ## The mask is for counting snps quickly, but trimming is still needed here
    ## to make the snps line up with the seqs in the snp string.
    snp = asnps[iloc, edg[0]:edg[1]+1, ]

    ## remove rows with all Ns, seq has only selected samples
    nalln = np.all(seq == "N", axis=1)

    ## make mask of removed rows and excluded samples. Use the inverse
    ## of this to save the coverage for samples
    nsidx = nalln + smask
    samplecov = samplecov + np.invert(nsidx).astype(np.int32)
    idx = np.sum(np.invert(nsidx).astype(np.int32))
    locuscov[idx] += 1

    ## select the remaining names in order
    seq = seq[~nsidx, ]
    names = pnames[~nsidx]

    ## save string for printing, excluding names not in samples
    outstr = "\n".join(\
        [name + s.tostring() for name, s in zip(names, seq)])

    ## get snp string and add to store
    snpstring = ["-" if snp[i, 0] else \
                 "*" if snp[i, 1] else \
                 " " for i in range(len(snp))]
    outstr += "\n" + snppad + "".join(snpstring) + "|{}|".format(iloc+start)
    #LOGGER.info("outstr %s", outstr)
    return outstr, samplecov, locuscov


Instruction:
Implement a Python 3 function for
creating array for final filtered snps data.

Response:
def init_arrays(data):
    """
    Create database file for storing final filtered snps data as hdf5 array.
    Copies splits and duplicates info from clust_database to database.
    """

    ## get stats from step6 h5 and create new h5
    co5 = h5py.File(data.clust_database, 'r')
    io5 = h5py.File(data.database, 'w')

    ## get maxlen and chunk len
    maxlen = data._hackersonly["max_fragment_length"] + 20
    chunks = co5["seqs"].attrs["chunksize"][0]
    nloci = co5["seqs"].shape[0]

    ## make array for snp string, 2 cols, - and *
    snps = io5.create_dataset("snps", (nloci, maxlen, 2),
                              dtype=np.bool,
                              chunks=(chunks, maxlen, 2),
                              compression='gzip')
    snps.attrs["chunksize"] = chunks
    snps.attrs["names"] = ["-", "*"]

    ## array for filters that will be applied in step7
    filters = io5.create_dataset("filters", (nloci, 6), dtype=np.bool)
    filters.attrs["filters"] = ["duplicates", "max_indels",
                                "max_snps", "max_shared_hets",
                                "min_samps", "max_alleles"]

    ## array for edgetrimming
    edges = io5.create_dataset("edges", (nloci, 5),
                               dtype=np.uint16,
                               chunks=(chunks, 5),
                               compression="gzip")
    edges.attrs["chunksize"] = chunks
    edges.attrs["names"] = ["R1_L", "R1_R", "R2_L", "R2_R", "sep"]

    ## xfer data from clustdb to finaldb
    edges[:, 4] = co5["splits"][:]
    filters[:, 0] = co5["duplicates"][:]

    ## close h5s
    io5.close()
    co5.close()


Instruction:
Can you generate the documentation for the following Python 3 function
def filter_stacks(data, sidx, hslice):
    LOGGER.info("Entering filter_stacks")

    ## open h5 handles
    io5 = h5py.File(data.clust_database, 'r')
    co5 = h5py.File(data.database, 'r')
    ## get a chunk (hslice) of loci for the selected samples (sidx)
    #superseqs = io5["seqs"][hslice[0]:hslice[1], sidx,]
    ## get an int view of the seq array
    #superints = io5["seqs"][hslice[0]:hslice[1], sidx, :].view(np.int8)

    ## we need to use upper to skip lowercase allele storage
    ## this slows down the rate of loading in data by a ton.
    superints = np.char.upper(io5["seqs"][hslice[0]:hslice[1], sidx,]).view(np.int8)
    LOGGER.info("superints shape {}".format(superints.shape))

    ## fill edge filter
    ## get edges of superseqs and supercats, since edges need to be trimmed
    ## before counting hets, snps, inds. Technically, this could edge trim
    ## clusters to the point that they are below the minlen, and so this
    ## also constitutes a filter, though one that is uncommon. For this
    ## reason we have another filter called edgfilter.
    splits = co5["edges"][hslice[0]:hslice[1], 4]
    edgfilter, edgearr = get_edges(data, superints, splits)
    del splits
    LOGGER.info('passed edges %s', hslice[0])

    ## minsamp coverages filtered from superseqs
    minfilter = filter_minsamp(data, superints)
    LOGGER.info('passed minfilt %s', hslice[0])

    ## maxhets per site column from superseqs after trimming edges
    hetfilter = filter_maxhet(data, superints, edgearr)
    LOGGER.info('passed minhet %s', hslice[0])

    ## ploidy filter
    pldfilter = io5["nalleles"][hslice[0]:hslice[1]].max(axis=1) > \
                                         data.paramsdict["max_alleles_consens"]

    ## indel filter, needs a fresh superints b/c get_edges does (-)->(N)
    indfilter = filter_indels(data, superints, edgearr)
    LOGGER.info('passed minind %s', hslice[0])

    ## Build the .loci snpstring as an array (snps)
    ## shape = (chunk, 1) dtype=S1, or should it be (chunk, 2) for [-,*] ?
    snpfilter, snpsarr = filter_maxsnp(data, superints, edgearr)

    LOGGER.info("edg %s", edgfilter.sum())
    LOGGER.info("min %s", minfilter.sum())
    LOGGER.info("het %s", hetfilter.sum())
    LOGGER.info("pld %s", pldfilter.sum())
    LOGGER.info("snp %s", snpfilter.sum())
    LOGGER.info("ind %s", indfilter.sum())

    ## SAVE FILTERS AND INFO TO DISK BY SLICE NUMBER (.0.tmp.h5)
    chunkdir = os.path.join(data.dirs.outfiles, data.name+"_tmpchunks")

    handle = os.path.join(chunkdir, "edgf.{}.npy".format(hslice[0]))
    with open(handle, 'w') as out:
        np.save(out, edgfilter)

    handle = os.path.join(chunkdir, "minf.{}.npy".format(hslice[0]))
    with open(handle, 'w') as out:
        np.save(out, minfilter)

    handle = os.path.join(chunkdir, "hetf.{}.npy".format(hslice[0]))
    with open(handle, 'w') as out:
        np.save(out, hetfilter)

    handle = os.path.join(chunkdir, "snpf.{}.npy".format(hslice[0]))
    with open(handle, 'w') as out:
        np.save(out, snpfilter)

    handle = os.path.join(chunkdir, "pldf.{}.npy".format(hslice[0]))
    with open(handle, 'w') as out:
        np.save(out, pldfilter)

    handle = os.path.join(chunkdir, "indf.{}.npy".format(hslice[0]))
    with open(handle, 'w') as out:
        np.save(out, indfilter)

    handle = os.path.join(chunkdir, "snpsarr.{}.npy".format(hslice[0]))
    with open(handle, 'w') as out:
        np.save(out, snpsarr)

    handle = os.path.join(chunkdir, "edgearr.{}.npy".format(hslice[0]))
    with open(handle, 'w') as out:
        np.save(out, edgearr)

    io5.close()
    co5.close()

Response: Filter the given stack of loci.


Instruction:
How would you implement a function in Python 3 that
gets the edges of the base sequence of the base sequence of the base sequence of the base sequence of the base sequence of the base sequence of the base sequence.

Response:
def get_edges(data, superints, splits):
    """
    Gets edge trimming based on the overlap of sequences at the edges of
    alignments and the tuple arg passed in for edge_trimming. Trims as
    (R1 left, R1 right, R2 left, R2 right). We also trim off the restriction
    site if it present. This modifies superints, and so should be run on an
    engine so it doesn't affect local copy. If this is changed to run locally
    for some reason make sure we copy the superints instead.
    """
    ## the filtering arg and parse it into minsamp numbers
    if "trim_overhang" in data.paramsdict:
        edgetrims = np.array(data.paramsdict["trim_overhang"]).astype(np.int16)
    else:
        edgetrims = np.array(data.paramsdict["trim_loci"]).astype(np.int16)

    ## Cuts 3 and 4 are only for 3rad/radcap
    ## TODO: This is moderately hackish, it's not using cut3/4
    ## correctly, just assuming the length is the same as cut1/2
    try:
        cut1, cut2, _, _ = data.paramsdict["restriction_overhang"]
        LOGGER.debug("Found 3Rad cut sites")
    except ValueError:
        cut1, cut2 = data.paramsdict["restriction_overhang"]

    cuts = np.array([len(cut1), len(cut2)], dtype=np.int16)

    ## a local array for storing edge trims
    edges = np.zeros((superints.shape[0], 5), dtype=np.int16)

    ## a local array for storing edge filtered loci, these are stored
    ## eventually as minsamp excludes.
    edgefilter = np.zeros((superints.shape[0],), dtype=np.bool)

    ## TRIM GUIDE. The cut site lengths are always trimmed. In addition,
    ## edge overhangs are trimmed to min(4, minsamp), and then additional
    ## number of columns is trimmed based on edgetrims values.
    ## A special case, -1 value means no trim at all.
    if data.paramsdict["min_samples_locus"] <= 4:
        minedge = np.int16(data.paramsdict["min_samples_locus"])
    else:
        minedge = np.int16(max(4, data.paramsdict["min_samples_locus"]))

    ## convert all - to N to make this easier
    nodashints = copy.deepcopy(superints)#.copy()
    nodashints[nodashints == 45] = 78

    ## trim overhanging edges
    ## get the number not Ns in each site,
    #ccx = np.sum(superseqs != "N", axis=1)
    ccx = np.sum(nodashints != 78, axis=1, dtype=np.uint16)
    efi, edg = edgetrim_numba(splits, ccx, edges, edgefilter, edgetrims, cuts, minedge)
    return efi, edg


Instruction:
Can you tell what is the following Python 3 function doing
def filter_minsamp(data, superints):
    ## global minsamp
    minsamp = data.paramsdict["min_samples_locus"]

    ## use population minsamps
    if data.populations:
        ## data._populations will look like this:
        ## {'a': (3, [0, 1, 2, 3],
        ##  'b': (3, [4, 5, 6, 7],
        ##  'c': (3, [8, 9, 10, 11]}
        LOGGER.info("POPULATIONS %s", data.populations)

        ## superints has already been subsampled by sidx
        ## get the sidx values for each pop
        minfilters = []
        for pop in data._populations:
            samps = data._populations[pop][1]
            minsamp = data._populations[pop][0]
            mini = np.sum(~np.all(superints[:, samps, :] == 78, axis=2), axis=1) < minsamp
            minfilters.append(mini)
        ## get sum across all pops for each locus
        minfilt = np.any(minfilters, axis=0)

    else:
        ## if not pop-file use global minsamp filter
        minfilt = np.sum(~np.all(superints == 78, axis=2), axis=1) < minsamp
        #LOGGER.info("Filtered by min_samples_locus - {}".format(minfilt.sum()))
    return minfilt

Response: Filter minimum samples per locus from superseqs [ chunk ].


Instruction:
Can you generate the documentation for the following Python 3 function
def ucount(sitecol):

    ## a list for only catgs
    catg = [i for i in sitecol if i in "CATG"]

    ## find sites that are ambigs
    where = [sitecol[sitecol == i] for i in "RSKYWM"]

    ## for each occurrence of RSKWYM add ambig resolution to catg
    for ambig in where:
        for _ in range(ambig.size):
            catg += list(AMBIGS[ambig[0]])

    ## if invariant return " "
    if len(set(catg)) < 2:
        return " "
    else:
        ## get second most common site
        second = Counter(catg).most_common()[1][1]
        if second > 1:
            return "*"
        else:
            return "-"

Response: Returns the number of unique bases in a site for a given site.


Instruction:
Can you implement a function in Python 3 that
filters max SNPs per locus and save in the array

Response:
def filter_maxsnp(data, superints, edgearr):
    """
    Filter max # of SNPs per locus. Do R1 and R2 separately if PE.
    Also generate the snpsite line for the .loci format and save in the snp arr
    This uses the edge filters that have been built based on trimming, and
    saves the snps array with edges filtered. **Loci are not yet filtered.**
    """

    ## an empty array to count with failed loci
    snpfilt = np.zeros(superints.shape[0], dtype=np.bool)
    snpsarr = np.zeros((superints.shape[0], superints.shape[2], 2), dtype=np.bool)
    maxsnps = np.array(data.paramsdict['max_SNPs_locus'], dtype=np.int16)

    ## get the per site snp string | shape=(chunk, maxlen)
    # snpsarr[:, :, 0] = snps == "-"
    # snpsarr[:, :, 1] = snps == "*"
    snpsarr = snpcount_numba(superints, snpsarr)
    LOGGER.info("---found the snps: %s", snpsarr.sum())
    snpfilt, snpsarr = snpfilter_numba(snpsarr, snpfilt, edgearr, maxsnps)
    LOGGER.info("---filtered snps: %s", snpfilt.sum())
    return snpfilt, snpsarr


Instruction:
Can you write a function in Python 3 where it
counts the number of unique bases in a site.

Response:
def snpcount_numba(superints, snpsarr):
    """
    Used to count the number of unique bases in a site for snpstring.
    """
    ## iterate over all loci
    for iloc in xrange(superints.shape[0]):
        for site in xrange(superints.shape[2]):

            ## make new array
            catg = np.zeros(4, dtype=np.int16)

            ## a list for only catgs
            ncol = superints[iloc, :, site]
            for idx in range(ncol.shape[0]):
                if ncol[idx] == 67: #C
                    catg[0] += 1
                elif ncol[idx] == 65: #A
                    catg[1] += 1
                elif ncol[idx] == 84: #T
                    catg[2] += 1
                elif ncol[idx] == 71: #G
                    catg[3] += 1
                elif ncol[idx] == 82: #R
                    catg[1] += 1        #A
                    catg[3] += 1        #G
                elif ncol[idx] == 75: #K
                    catg[2] += 1        #T
                    catg[3] += 1        #G
                elif ncol[idx] == 83: #S
                    catg[0] += 1        #C
                    catg[3] += 1        #G
                elif ncol[idx] == 89: #Y
                    catg[0] += 1        #C
                    catg[2] += 1        #T
                elif ncol[idx] == 87: #W
                    catg[1] += 1        #A
                    catg[2] += 1        #T
                elif ncol[idx] == 77: #M
                    catg[0] += 1        #C
                    catg[1] += 1        #A

            ## get second most common site
            catg.sort()
            ## if invariant e.g., [0, 0, 0, 9], then nothing (" ")
            if not catg[2]:
                pass
            else:
                if catg[2] > 1:
                    snpsarr[iloc, site, 1] = True
                else:
                    snpsarr[iloc, site, 0] = True
    return snpsarr


Instruction:
Explain what the following Python 3 code does
def filter_maxhet(data, superints, edgearr):
    ## the filter max
    ## The type of max_shared_Hs_locus is determined and the cast to either
    ## int or float is made at assembly load time
    maxhet = data.paramsdict["max_shared_Hs_locus"]
    if isinstance(maxhet, float):
        ## get an array with maxhet fraction * ntaxa with data for each locus
        #maxhet = np.array(superints.shape[1]*maxhet, dtype=np.int16)
        maxhet = np.floor(
            maxhet * (superints.shape[1] -
                np.all(superints == 78, axis=2).sum(axis=1))).astype(np.int16)
    elif isinstance(maxhet, int):
        maxhet = np.zeros(superints.shape[0], dtype=np.int16)
        maxhet.fill(data.paramsdict["max_shared_Hs_locus"])

    ## an empty array to fill with failed loci
    LOGGER.info("--------------maxhet mins %s", maxhet)
    hetfilt = np.zeros(superints.shape[0], dtype=np.bool)
    hetfilt = maxhet_numba(superints, edgearr, maxhet, hetfilt)
    LOGGER.info("--------------maxhet sums %s", hetfilt.sum())
    return hetfilt

Response: Filter maxheterozygosity per locus.


Instruction:
Can you generate the documentation for the following Python 3 function
def filter_indels(data, superints, edgearr):

    maxinds = np.array(data.paramsdict["max_Indels_locus"]).astype(np.int64)

    ## an empty array to fill with failed loci
    ifilter = np.zeros(superints.shape[0], dtype=np.bool_)

    ## if paired then worry about splits
    if "pair" in data.paramsdict["datatype"]:
        for idx in xrange(superints.shape[0]):
            block1 = superints[idx, :, edgearr[idx, 0]:edgearr[idx, 1]]
            block2 = superints[idx, :, edgearr[idx, 2]:edgearr[idx, 3]]

            sums1 = maxind_numba(block1)
            ## If all loci are merged then block2 will be empty which will
            ## cause maxind_numba to throw a very confusing ValueError
            if np.any(block2):
                sums2 = maxind_numba(block2)
            else:
                sums2 = 0

            if (sums1 > maxinds[0]) or (sums2 > maxinds[1]):
                ifilter[idx] = True

    else:
        for idx in xrange(superints.shape[0]):
            ## get block based on edge filters
            block = superints[idx, :, edgearr[idx, 0]:edgearr[idx, 1]]
            ## shorten block to exclude terminal indels
            ## if data at this locus (not already filtered by edges/minsamp)
            if block.shape[1] > 1:
                try:
                    sums = maxind_numba(block)
                except ValueError as inst:
                    msg = "All loci filterd by max_Indels_locus. Try increasing this parameter value."
                    raise IPyradError(msg)
                except Exception as inst:
                    LOGGER.error("error in block {}".format(block))
                #LOGGER.info("maxind numba %s %s", idx, sums)
                #LOGGER.info("sums, maxinds[0], compare: %s %s %s",
                #             sums, maxinds[0], sums > maxinds[0])
                if sums > maxinds[0]:
                    ifilter[idx] = True

    LOGGER.info("--------------maxIndels sums %s", ifilter.sum())
    return ifilter

Response: Filter out all indels that are not in the max_Indels_locus parameter.


Instruction:
Write a Python 3 script to
filter for indels that are not 45

Response:
def maxind_numba(block):
    """ filter for indels """
    ## remove terminal edges
    inds = 0
    for row in xrange(block.shape[0]):
        where = np.where(block[row] != 45)[0]
        if len(where) == 0:
            obs = 100
        else:
            left = np.min(where)
            right = np.max(where)
            obs = np.sum(block[row, left:right] == 45)
        if obs > inds:
            inds = obs
    return inds


Instruction:
How would you explain what the following Python 3 function does
def make_outfiles(data, samples, output_formats, ipyclient):

    ## will iterate optim loci at a time
    with h5py.File(data.clust_database, 'r') as io5:
        optim = io5["seqs"].attrs["chunksize"][0]
        nloci = io5["seqs"].shape[0]

        ## get name and snp padding
        anames = io5["seqs"].attrs["samples"]
        snames = [i.name for i in samples]
        ## get only snames in this data set sorted in the order they are in io5
        names = [i for i in anames if i in snames]
        pnames, _ = padnames(names)

    ## get names boolean
    sidx = np.array([i in snames for i in anames])
    assert len(pnames) == sum(sidx)

    ## get names index in order of pnames
    #sindx = [list(anames).index(i) for i in snames]

    ## send off outputs as parallel jobs
    lbview = ipyclient.load_balanced_view()
    start = time.time()
    results = {}

    ## build arrays and outputs from arrays.
    ## these arrays are keys in the tmp h5 array: seqarr, snparr, bisarr, maparr
    boss_make_arrays(data, sidx, optim, nloci, ipyclient)

    start = time.time()
    ## phy and partitions are a default output ({}.phy, {}.phy.partitions)
    if "p" in output_formats:
        data.outfiles.phy = os.path.join(data.dirs.outfiles, data.name+".phy")
        async = lbview.apply(write_phy, *[data, sidx, pnames])
        results['phy'] = async

    ## nexus format includes ... additional information ({}.nex)
    if "n" in output_formats:
        data.outfiles.nexus = os.path.join(data.dirs.outfiles, data.name+".nex")
        async = lbview.apply(write_nex, *[data, sidx, pnames])
        results['nexus'] = async

    ## snps is actually all snps written in phylip format ({}.snps.phy)
    if "s" in output_formats:
        data.outfiles.snpsmap = os.path.join(data.dirs.outfiles, data.name+".snps.map")
        data.outfiles.snpsphy = os.path.join(data.dirs.outfiles, data.name+".snps.phy")
        async = lbview.apply(write_snps, *[data, sidx, pnames])
        results['snps'] = async
        async = lbview.apply(write_snps_map, data)
        results['snpsmap'] = async

    ## usnps is one randomly sampled snp from each locus ({}.u.snps.phy)
    if "u" in output_formats:
        data.outfiles.usnpsphy = os.path.join(data.dirs.outfiles, data.name+".u.snps.phy")
        async = lbview.apply(write_usnps, *[data, sidx, pnames])
        results['usnps'] = async

    ## str and ustr are for structure analyses. A fairly outdated format, six
    ## columns of empty space. Full and subsample included ({}.str, {}.u.str)
    if "k" in output_formats:
        data.outfiles.str = os.path.join(data.dirs.outfiles, data.name+".str")
        data.outfiles.ustr = os.path.join(data.dirs.outfiles, data.name+".ustr")
        async = lbview.apply(write_str, *[data, sidx, pnames])
        results['structure'] = async

    ## geno output is for admixture and other software. We include all SNPs,
    ## but also a .map file which has "distances" between SNPs.
    if 'g' in output_formats:
        data.outfiles.geno = os.path.join(data.dirs.outfiles, data.name+".geno")
        data.outfiles.ugeno = os.path.join(data.dirs.outfiles, data.name+".u.geno")
        async = lbview.apply(write_geno, *[data, sidx])
        results['geno'] = async

    ## G-PhoCS output. Have to use cap G here cuz little g is already taken, lol.
    if 'G' in output_formats:
        data.outfiles.gphocs = os.path.join(data.dirs.outfiles, data.name+".gphocs")
        async = lbview.apply(write_gphocs, *[data, sidx])
        results['gphocs'] = async

    ## wait for finished outfiles
    while 1:
        readies = [i.ready() for i in results.values()]
        elapsed = datetime.timedelta(seconds=int(time.time()-start))
        progressbar(len(readies), sum(readies),
            " writing outfiles      | {} | s7 |".format(elapsed),
            spacer=data._spacer)
        time.sleep(0.1)
        if all(readies):
            break
    print("")

    ## check for errors
    for suff, async in results.items():
        if not async.successful():
            print("  Warning: error encountered while writing {} outfile: {}"\
                  .format(suff, async.exception()))
            LOGGER.error("  Warning: error in writing %s outfile: %s", \
                         suff, async.exception())

    ## remove the tmparrays
    tmparrs = os.path.join(data.dirs.outfiles, "tmp-{}.h5".format(data.name))
    os.remove(tmparrs)

Response: Create output files for all samples in the data set.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def worker_make_arrays(data, sidx, hslice, optim, maxlen):

    ## big data arrays
    io5 = h5py.File(data.clust_database, 'r')
    co5 = h5py.File(data.database, 'r')

    ## temporary storage until writing to h5 array
    maxsnp = co5["snps"][hslice:hslice+optim].sum()         ## concat later
    maparr = np.zeros((maxsnp, 4), dtype=np.uint32)
    snparr = np.zeros((sum(sidx), maxsnp), dtype="S1")
    bisarr = np.zeros((sum(sidx), maxsnp), dtype="S1")
    seqarr = np.zeros((sum(sidx), maxlen*optim), dtype="S1")

    ## apply all filters and write loci data
    seqleft = 0
    snpleft = 0
    bis = 0

    ## edge filter has already been applied to snps, but has not yet been
    ## applied to seqs. The locus filters have not been applied to either yet.
    mapsnp = 0
    totloc = 0

    afilt = co5["filters"][hslice:hslice+optim, :]
    aedge = co5["edges"][hslice:hslice+optim, :]
    asnps = co5["snps"][hslice:hslice+optim, :]
    #aseqs = io5["seqs"][hslice:hslice+optim, sidx, :]
    ## have to run upper on seqs b/c they have lowercase storage of alleles
    aseqs = np.char.upper(io5["seqs"][hslice:hslice+optim, sidx, :])

    ## which loci passed all filters
    keep = np.where(np.sum(afilt, axis=1) == 0)[0]

    ## write loci that passed after trimming edges, then write snp string
    for iloc in keep:
        ## grab r1 seqs between edges
        edg = aedge[iloc]

        ## grab SNPs from seqs already sidx subsampled and edg masked.
        ## needs to be done here before seqs are edgetrimmed.
        getsnps = asnps[iloc].sum(axis=1).astype(np.bool)
        snps = aseqs[iloc, :, getsnps].T

        ## trim edges and split from seqs and concatenate for pairs.
        ## this seq array will be the phy output.
        if not "pair" in data.paramsdict["datatype"]:
            seq = aseqs[iloc, :, edg[0]:edg[1]+1]
        else:
            seq = np.concatenate([aseqs[iloc, :, edg[0]:edg[1]+1],
                                  aseqs[iloc, :, edg[2]:edg[3]+1]], axis=1)

        ## remove cols from seq (phy) array that are all N-
        lcopy = seq
        lcopy[lcopy == "-"] = "N"
        bcols = np.all(lcopy == "N", axis=0)
        seq = seq[:, ~bcols]

        ## put into large array (could put right into h5?)
        seqarr[:, seqleft:seqleft+seq.shape[1]] = seq
        seqleft += seq.shape[1]

        ## subsample all SNPs into an array
        snparr[:, snpleft:snpleft+snps.shape[1]] = snps
        snpleft += snps.shape[1]

        ## Enter each snp into the map file
        for i in xrange(snps.shape[1]):
            ## 1-indexed loci in first column
            ## actual locus number in second column
            ## counter for this locus in third column
            ## snp counter total in fourth column
            maparr[mapsnp, :] = [totloc+1, hslice+iloc, i, mapsnp+1]
            mapsnp += 1

        ## subsample one SNP into an array
        if snps.shape[1]:
            samp = np.random.randint(snps.shape[1])
            bisarr[:, bis] = snps[:, samp]
            bis += 1
            totloc += 1

    ## clean up
    io5.close()
    co5.close()

    ## trim trailing edges b/c we made the array bigger than needed.
    ridx = np.all(seqarr == "", axis=0)
    seqarr = seqarr[:, ~ridx]
    ridx = np.all(snparr == "", axis=0)
    snparr = snparr[:, ~ridx]
    ridx = np.all(bisarr == "", axis=0)
    bisarr = bisarr[:, ~ridx]
    ridx = np.all(maparr == 0, axis=1)
    maparr = maparr[~ridx, :]

    ## return these three arrays which are pretty small
    ## catg array gets to be pretty huge, so we return only
    return seqarr, snparr, bisarr, maparr

Response: Parallelized worker to build array chunks for output files.


Instruction:
How would you implement a function in Python 3 that
writes the phylip output file from the tmparr [ seqarray ]

Response:
def write_phy(data, sidx, pnames):
    """
    write the phylip output file from the tmparr[seqarray]
    """

    ## grab seq data from tmparr
    start = time.time()
    tmparrs = os.path.join(data.dirs.outfiles, "tmp-{}.h5".format(data.name))
    with h5py.File(tmparrs, 'r') as io5:
        seqarr = io5["seqarr"]

        ## trim to size b/c it was made longer than actual
        end = np.where(np.all(seqarr[:] == "", axis=0))[0]
        if np.any(end):
            end = end.min()
        else:
            end = seqarr.shape[1]

        ## write to phylip
        with open(data.outfiles.phy, 'w') as out:
            ## write header
            out.write("{} {}\n".format(seqarr.shape[0], end))

            ## write data rows
            for idx, name in enumerate(pnames):
                out.write("{}{}\n".format(name, "".join(seqarr[idx, :end])))
    LOGGER.debug("finished writing phy in: %s", time.time() - start)


Instruction:
Here you have a function in Python 3, explain what it does
def write_nex(data, sidx, pnames):

    ## grab seq data from tmparr
    start = time.time()
    tmparrs = os.path.join(data.dirs.outfiles, "tmp-{}.h5".format(data.name))
    with h5py.File(tmparrs, 'r') as io5:
        seqarr = io5["seqarr"]

        ## trim to size b/c it was made longer than actual
        end = np.where(np.all(seqarr[:] == "", axis=0))[0]
        if np.any(end):
            end = end.min()
        else:
            end = seqarr.shape[1]

        ## write to nexus
        data.outfiles.nex = os.path.join(data.dirs.outfiles, data.name+".nex")

        with open(data.outfiles.nex, 'w') as out:

            ## write nexus seq header
            out.write(NEXHEADER.format(seqarr.shape[0], end))

            ## grab a big block of data
            chunksize = 100000  # this should be a multiple of 100
            for bidx in xrange(0, end, chunksize):
                bigblock = seqarr[:, bidx:bidx+chunksize]
                lend = end-bidx
                #LOGGER.info("BIG: %s %s %s %s", bigblock.shape, bidx, lend, end)

                ## write interleaved seqs 100 chars with longname+2 before
                tmpout = []
                for block in xrange(0, min(chunksize, lend), 100):
                    stop = min(block+100, end)

                    for idx, name in enumerate(pnames):
                        seqdat = bigblock[idx, block:stop]
                        tmpout.append("  {}{}\n".format(name, "".join(seqdat)))
                    tmpout.append("\n")

                ## print intermediate result and clear
                if any(tmpout):
                    out.write("".join(tmpout))
            ## closer
            out.write(NEXCLOSER)
    LOGGER.debug("finished writing nex in: %s", time.time() - start)

Response: write the nexus output file from the tmparr seqarray and maparr


Instruction:
Create a Python 3 function for
writing a map file with linkage information for SNPs file

Response:
def write_snps_map(data):
    """ write a map file with linkage information for SNPs file"""

    ## grab map data from tmparr
    start = time.time()
    tmparrs = os.path.join(data.dirs.outfiles, "tmp-{}.h5".format(data.name))
    with h5py.File(tmparrs, 'r') as io5:
        maparr = io5["maparr"][:]

        ## get last data
        end = np.where(np.all(maparr[:] == 0, axis=1))[0]
        if np.any(end):
            end = end.min()
        else:
            end = maparr.shape[0]

        ## write to map file (this is too slow...)
        outchunk = []
        with open(data.outfiles.snpsmap, 'w') as out:
            for idx in xrange(end):
                ## build to list
                line = maparr[idx, :]
                #print(line)
                outchunk.append(\
                    "{}\trad{}_snp{}\t{}\t{}\n"\
                    .format(line[0], line[1], line[2], 0, line[3]))
                ## clear list
                if not idx % 10000:
                    out.write("".join(outchunk))
                    outchunk = []
            ## write remaining
            out.write("".join(outchunk))
    LOGGER.debug("finished writing snps_map in: %s", time.time() - start)


Instruction:
Create a Python 3 function for
writing the bisnp string

Response:
def write_usnps(data, sidx, pnames):
    """ write the bisnp string """

    ## grab bis data from tmparr
    tmparrs = os.path.join(data.dirs.outfiles, "tmp-{}.h5".format(data.name))
    with h5py.File(tmparrs, 'r') as io5:
        bisarr = io5["bisarr"]

        ## trim to size b/c it was made longer than actual
        end = np.where(np.all(bisarr[:] == "", axis=0))[0]
        if np.any(end):
            end = end.min()
        else:
            end = bisarr.shape[1]

        ## write to usnps file
        with open(data.outfiles.usnpsphy, 'w') as out:
            out.write("{} {}\n".format(bisarr.shape[0], end))
            for idx, name in enumerate(pnames):
                out.write("{}{}\n".format(name, "".join(bisarr[idx, :end])))


Instruction:
Explain what the following Python 3 code does
def write_str(data, sidx, pnames):

    ## grab snp and bis data from tmparr
    start = time.time()
    tmparrs = os.path.join(data.dirs.outfiles, "tmp-{}.h5".format(data.name))
    with h5py.File(tmparrs, 'r') as io5:
        snparr = io5["snparr"]
        bisarr = io5["bisarr"]

        ## trim to size b/c it was made longer than actual
        bend = np.where(np.all(bisarr[:] == "", axis=0))[0]
        if np.any(bend):
            bend = bend.min()
        else:
            bend = bisarr.shape[1]

        send = np.where(np.all(snparr[:] == "", axis=0))[0]
        if np.any(send):
            send = send.min()
        else:
            send = snparr.shape[1]

        ## write to str and ustr
        out1 = open(data.outfiles.str, 'w')
        out2 = open(data.outfiles.ustr, 'w')
        numdict = {'A': '0', 'T': '1', 'G': '2', 'C': '3', 'N': '-9', '-': '-9'}
        if data.paramsdict["max_alleles_consens"] > 1:
            for idx, name in enumerate(pnames):
                out1.write("{}\t\t\t\t\t{}\n"\
                    .format(name,
                    "\t".join([numdict[DUCT[i][0]] for i in snparr[idx, :send]])))
                out1.write("{}\t\t\t\t\t{}\n"\
                    .format(name,
                    "\t".join([numdict[DUCT[i][1]] for i in snparr[idx, :send]])))
                out2.write("{}\t\t\t\t\t{}\n"\
                    .format(name,
                    "\t".join([numdict[DUCT[i][0]] for i in bisarr[idx, :bend]])))
                out2.write("{}\t\t\t\t\t{}\n"\
                    .format(name,
                    "\t".join([numdict[DUCT[i][1]] for i in bisarr[idx, :bend]])))
        else:
            ## haploid output
            for idx, name in enumerate(pnames):
                out1.write("{}\t\t\t\t\t{}\n"\
                    .format(name,
                    "\t".join([numdict[DUCT[i][0]] for i in snparr[idx, :send]])))
                out2.write("{}\t\t\t\t\t{}\n"\
                    .format(name,
                    "\t".join([numdict[DUCT[i][0]] for i in bisarr[idx, :bend]])))
        out1.close()
        out2.close()
    LOGGER.debug("finished writing str in: %s", time.time() - start)

Response: Write STRUCTURE format for all SNPs and unlinked SNPs


Instruction:
Can you tell what is the following Python 3 function doing
def write_geno(data, sidx):

    ## grab snp and bis data from tmparr
    start = time.time()
    tmparrs = os.path.join(data.dirs.outfiles, "tmp-{}.h5".format(data.name))
    with h5py.File(tmparrs, 'r') as io5:
        snparr = io5["snparr"]
        bisarr = io5["bisarr"]

        ## trim to size b/c it was made longer than actual
        bend = np.where(np.all(bisarr[:] == "", axis=0))[0]
        if np.any(bend):
            bend = bend.min()
        else:
            bend = bisarr.shape[1]

        send = np.where(np.all(snparr[:] == "", axis=0))[0]
        if np.any(send):
            send = send.min()
        else:
            send = snparr.shape[1]

        ## get most common base at each SNP as a pseudo-reference
        ## and record 0,1,2 or missing=9 for counts of the ref allele
        snpref = reftrick(snparr[:, :send].view(np.int8), GETCONS).view("S1")
        bisref = reftrick(bisarr[:, :bend].view(np.int8), GETCONS).view("S1")

        ## geno matrix to fill (9 is empty)
        snpgeno = np.zeros((snparr.shape[0], send), dtype=np.uint8)
        snpgeno.fill(9)
        bisgeno = np.zeros((bisarr.shape[0], bend), dtype=np.uint8)
        bisgeno.fill(9)

        ##--------------------------------------------------------------------
        ## fill in complete hits (match to first column ref base)
        mask2 = np.array(snparr[:, :send] == snpref[:, 0])
        snpgeno[mask2] = 2

        ## fill in single hits (heteros) match to hetero of first+second column
        ambref = np.apply_along_axis(lambda x: TRANSFULL[tuple(x)], 1, snpref[:, :2])
        mask1 = np.array(snparr[:, :send] == ambref)
        snpgeno[mask1] = 1

        ## fill in zero hits, meaning a perfect match to the second column base
        ## anything else is left at 9 (missing), b/c it's either missing or it
        ## is not bi-allelic.
        mask0 = np.array(snparr[:, :send] == snpref[:, 1])
        snpgeno[mask0] = 0

        ##--------------------------------------------------------------------

        ## fill in complete hits
        mask2 = np.array(bisarr[:, :bend] == bisref[:, 0])
        bisgeno[mask2] = 2

        ## fill in single hits (heteros)
        ambref = np.apply_along_axis(lambda x: TRANSFULL[tuple(x)], 1, bisref[:, :2])
        mask1 = np.array(bisarr[:, :bend] == ambref)
        bisgeno[mask1] = 1

        ## fill in zero hits (match to second base)
        mask0 = np.array(bisarr[:, :bend] == bisref[:, 1])
        bisgeno[mask0] = 0

        ##---------------------------------------------------------------------
        ## print to files
        np.savetxt(data.outfiles.geno, snpgeno.T, delimiter="", fmt="%d")
        np.savetxt(data.outfiles.ugeno, bisgeno.T, delimiter="", fmt="%d")
    LOGGER.debug("finished writing geno in: %s", time.time() - start)

Response: write the geno output for the given sNMF


Instruction:
How would you implement a function in Python 3 that
writes the g - phocs output for a given sample index

Response:
def write_gphocs(data, sidx):
    """
    write the g-phocs output. This code is hella ugly bcz it's copy/pasted
    directly from the old loci2gphocs script from pyrad. I figure having it
    get done the stupid way is better than not having it done at all, at
    least for the time being. This could probably be sped up significantly.
    """

    outfile = data.outfiles.gphocs
    infile = data.outfiles.loci

    infile = open(infile)
    outfile = open(outfile, 'w')

    ## parse the loci
    ## Each set of reads at a locus is appended with a line
    ## beginning with // and ending with |x, where x in the locus id.
    ## so after this call 'loci' will contain an array
    ## of sets of each read per locus.
    loci = re.compile("\|[0-9]+\|").split(infile.read())[:-1]

    # Print the header, the number of loci in this file
    outfile.write(str(len(loci)) + "\n\n")

    # iterate through each locus, print out the header for each locus:
    # <locus_name> <n_samples> <locus_length>
    # Then print the data for each sample in this format:
    # <individual_name> <sequence>
    for i, loc in enumerate(loci):
        ## Get rid of the line that contains the snp info
        loc = loc.rsplit("\n", 1)[0]

        # Separate out each sequence within the loc block. 'sequences'
        # will now be a list strings containing name/sequence pairs.
        # We select each line in the locus string that starts with ">"
        names = [line.split()[0] for line in loc.strip().split("\n")]
        try:
            sequences = [line.split()[1] for line in loc.strip().split("\n")]
        except:
            pass
        # Strips off 'nnnn' separator for paired data
        # replaces '-' with 'N'
        editsequences = [seq.replace("n","").replace('-','N') for seq in sequences]
        sequence_length = len(editsequences[0])

        # get length of longest name and add 4 spaces
        longname = max(map(len,names))+4

        # Print out the header for this locus
        outfile.write('locus{} {} {}\n'.format(str(i), len(sequences), sequence_length))

        # Iterate through each sequence read at this locus and write it to the file.
        for name,sequence in zip(names, editsequences):
            # Clean up the sequence data to make gphocs happy. Only accepts UPPER
            # case chars for bases, and only accepts 'N' for missing data.
            outfile.write(name+" "*(longname-len(name))+sequence + "\n")
        ## Separate loci with so it's prettier
        outfile.write("\n")


Instruction:
Can you write a function in Python 3 where it
creates a vcf file for the loci passing filtering.

Response:
def make_vcf(data, samples, ipyclient, full=0):
    """
    Write the full VCF for loci passing filtering. Other vcf formats are
    possible, like SNPs-only, or with filtered loci included but the filter
    explicitly labeled. These are not yet supported, however.
    """
    ## start vcf progress bar
    start = time.time()
    printstr = " building vcf file     | {} | s7 |"
    LOGGER.info("Writing .vcf file")
    elapsed = datetime.timedelta(seconds=int(time.time()-start))
    progressbar(20, 0, printstr.format(elapsed), spacer=data._spacer)

    ## create outputs for v and V, gzip V to be friendly
    data.outfiles.vcf = os.path.join(data.dirs.outfiles, data.name+".vcf")
    if full:
        data.outfiles.VCF = os.path.join(data.dirs.outfiles, data.name+".vcf.gz")

    ## get some db info
    with h5py.File(data.clust_database, 'r') as io5:
        ## will iterate optim loci at a time
        optim = io5["seqs"].attrs["chunksize"][0]
        nloci = io5["seqs"].shape[0]
        ## get name and snp padding
        anames = io5["seqs"].attrs["samples"]
        snames = [i.name for i in samples]
        names = [i for i in anames if i in snames]

    ## get names index
    sidx = np.array([i in snames for i in anames])

    ## client for sending jobs to parallel engines, for this step we'll limit
    ## to half of the available cpus if
    lbview = ipyclient.load_balanced_view()

    ## send jobs in chunks
    vasyncs = {}
    total = 0
    for chunk in xrange(0, nloci, optim):
        vasyncs[chunk] = lbview.apply(vcfchunk, *(data, optim, sidx, chunk, full))
        total += 1

    ## tmp files get left behind and intensive processes are left running when a
    ## a job is killed/interrupted during vcf build, so we try/except wrap.
    try:
        while 1:
            keys = [i for (i, j) in vasyncs.items() if j.ready()]
            ## check for failures
            for job in keys:
                if not vasyncs[job].successful():
                    ## raise exception
                    err = " error in vcf build chunk {}: {}"\
                          .format(job, vasyncs[job].result())
                    LOGGER.error(err)
                    raise IPyradWarningExit(err)
                else:
                    ## free up memory
                    del vasyncs[job]

            finished = total - len(vasyncs) #sum([i.ready() for i in vasyncs.values()])
            elapsed = datetime.timedelta(seconds=int(time.time()-start))
            progressbar(total, finished, printstr.format(elapsed), spacer=data._spacer)
            time.sleep(0.5)
            if not vasyncs:
                break
        print("")

    except Exception as inst:
        ## make sure all future jobs are aborted
        keys = [i for (i, j) in vasyncs.items() if not j.ready()]
        try:
            for job in keys:
                #vasyncs[job].abort()
                vasyncs[job].cancel()
        except Exception:
            pass
        ## make sure all tmp files are destroyed
        vcfchunks = glob.glob(os.path.join(data.dirs.outfiles, "*.vcf.[0-9]*"))
        h5chunks = glob.glob(os.path.join(data.dirs.outfiles, ".tmp.[0-9]*.h5"))
        for dfile in vcfchunks+h5chunks:
            os.remove(dfile)
        ## reraise the error
        raise inst

    ## writing full vcf file to disk
    start = time.time()
    printstr = " writing vcf file      | {} | s7 |"
    res = lbview.apply(concat_vcf, *(data, names, full))
    ogchunks = len(glob.glob(data.outfiles.vcf+".*"))
    while 1:
        elapsed = datetime.timedelta(seconds=int(time.time()-start))
        curchunks = len(glob.glob(data.outfiles.vcf+".*"))
        progressbar(ogchunks, ogchunks-curchunks, printstr.format(elapsed), spacer=data._spacer)
        time.sleep(0.1)
        if res.ready():
            break
    elapsed = datetime.timedelta(seconds=int(time.time()-start))
    progressbar(1, 1, printstr.format(elapsed), spacer=data._spacer)
    print("")


Instruction:
Write a Python 3 function that can
concatenate a VCF file into a single file.

Response:
def concat_vcf(data, names, full):
    """
    Sorts, concatenates, and gzips VCF chunks. Also cleans up chunks.
    """
    ## open handle and write headers
    if not full:
        writer = open(data.outfiles.vcf, 'w')
    else:
        writer = gzip.open(data.outfiles.VCF, 'w')
    vcfheader(data, names, writer)
    writer.close()

    ## get vcf chunks
    vcfchunks = glob.glob(data.outfiles.vcf+".*")
    vcfchunks.sort(key=lambda x: int(x.rsplit(".")[-1]))

    ## concatenate
    if not full:
        writer = open(data.outfiles.vcf, 'a')
    else:
        writer = gzip.open(data.outfiles.VCF, 'a')

    ## what order do users want? The order in the original ref file?
    ## Sorted by the size of chroms? that is the order in faidx.
    ## If reference mapping then it's nice to sort the vcf data by
    ## CHROM and POS. This is doing a very naive sort right now, so the
    ## CHROM will be ordered, but not the pos within each chrom.
    if data.paramsdict["assembly_method"] in ["reference", "denovo+reference"]:
        ## Some unix sorting magic to get POS sorted within CHROM
        ## First you sort by POS (-k 2,2), then you do a `stable` sort
        ## by CHROM. You end up with POS ordered and grouped correctly by CHROM
        ## but relatively unordered CHROMs (locus105 will be before locus11).
        cmd = ["cat"] + vcfchunks + [" | sort -k 2,2 -n | sort -k 1,1 -s"]
        cmd = " ".join(cmd)
        proc = sps.Popen(cmd, shell=True, stderr=sps.STDOUT, stdout=writer, close_fds=True)
    else:
        proc = sps.Popen(["cat"] + vcfchunks, stderr=sps.STDOUT, stdout=writer, close_fds=True)

    err = proc.communicate()[0]
    if proc.returncode:
        raise IPyradWarningExit("err in concat_vcf: %s", err)
    writer.close()

    for chunk in vcfchunks:
        os.remove(chunk)


Instruction:
Write a Python 3 script for
functioning called within make_vcf to run a single chunk of vcf files

Response:
def vcfchunk(data, optim, sidx, chunk, full):
    """
    Function called within make_vcf to run chunks on separate engines.
    """
    ## empty array to be filled before writing
    ## will not actually be optim*maxlen, extra needs to be trimmed
    maxlen = data._hackersonly["max_fragment_length"] + 20

    ## get data sliced (optim chunks at a time)
    hslice = [chunk, chunk+optim]

    ## read all taxa from disk (faster), then subsample taxa with sidx and
    ## keepmask to greatly reduce the memory load
    with h5py.File(data.database, 'r') as co5:
        afilt = co5["filters"][hslice[0]:hslice[1], :]
        keepmask = afilt.sum(axis=1) == 0
        ## apply mask to edges
        aedge = co5["edges"][hslice[0]:hslice[1], :]
        aedge = aedge[keepmask, :]
    del afilt
    ## same memory subsampling.
    with h5py.File(data.clust_database, 'r') as io5:
        ## apply mask to edges to aseqs and acatg
        #aseqs = io5["seqs"][hslice[0]:hslice[1], :, :].view(np.uint8)
        ## need to read in seqs with upper b/c lowercase allele info
        aseqs = np.char.upper(io5["seqs"][hslice[0]:hslice[1], :, :]).view(np.uint8)
        aseqs = aseqs[keepmask, :]
        aseqs = aseqs[:, sidx, :]
        acatg = io5["catgs"][hslice[0]:hslice[1], :, :, :]
        acatg = acatg[keepmask, :]
        acatg = acatg[:, sidx, :, :]
        achrom = io5["chroms"][hslice[0]:hslice[1]]
        achrom = achrom[keepmask, :]

    LOGGER.info('acatg.shape %s', acatg.shape)
    ## to save memory some columns are stored in diff dtypes until printing
    if not full:
        with h5py.File(data.database, 'r') as co5:
            snps = co5["snps"][hslice[0]:hslice[1], :]
            snps = snps[keepmask, :]
            snps = snps.sum(axis=2)
        snpidxs = snps > 0
        maxsnplen = snps.sum()

    ## vcf info to fill, this is bigger than the actual array
    nrows = maxsnplen
    cols0 = np.zeros(nrows, dtype=np.int64) #h5py.special_dtype(vlen=bytes))
    cols1 = np.zeros(nrows, dtype=np.uint32)
    cols34 = np.zeros((nrows, 2), dtype="S5")
    cols7 = np.zeros((nrows, 1), dtype="S20")

    ## when nsamples is high this blows up memory (e.g., dim=(5M x 500))
    ## so we'll instead create a list of arrays with 10 samples at a time.
    ## maybe later replace this with a h5 array
    tmph = os.path.join(data.dirs.outfiles, ".tmp.{}.h5".format(hslice[0]))
    htmp = h5py.File(tmph, 'w')
    htmp.create_dataset("vcf", shape=(nrows, sum(sidx)), dtype="S24")

    ## which loci passed all filters
    init = 0

    ## write loci that passed after trimming edges, then write snp string
    locindex = np.where(keepmask)[0]
    for iloc in xrange(aseqs.shape[0]):
        edg = aedge[iloc]
        ## grab all seqs between edges
        if not 'pair' in data.paramsdict["datatype"]:
            seq = aseqs[iloc, :, edg[0]:edg[1]+1]
            catg = acatg[iloc, :, edg[0]:edg[1]+1]
            if not full:
                snpidx = snpidxs[iloc, edg[0]:edg[1]+1]
                seq = seq[:, snpidx]
                catg = catg[:, snpidx]
        else:
            seq = np.hstack([aseqs[iloc, :, edg[0]:edg[1]+1],
                             aseqs[iloc, :, edg[2]:edg[3]+1]])
            catg = np.hstack([acatg[iloc, :, edg[0]:edg[1]+1],
                              acatg[iloc, :, edg[2]:edg[3]+1]])
            if not full:
                snpidx = np.hstack([snpidxs[iloc, edg[0]:edg[1]+1],
                                    snpidxs[iloc, edg[2]:edg[3]+1]])
                seq = seq[:, snpidx]
                catg = catg[:, snpidx]

        ## empty arrs to fill
        alleles = np.zeros((nrows, 4), dtype=np.uint8)
        genos = np.zeros((seq.shape[1], sum(sidx)), dtype="S4")
        genos[:] = "./.:"

        ## ----  build string array ----
        pos = 0
        ## If any < 0 this indicates an anonymous locus in denovo+ref assembly
        if achrom[iloc][0] > 0:
            pos = achrom[iloc][1]
            cols0[init:init+seq.shape[1]] = achrom[iloc][0]
            cols1[init:init+seq.shape[1]] = pos + np.where(snpidx)[0] + 1
        else:
            if full:
                cols1[init:init+seq.shape[1]] = pos + np.arange(seq.shape[1]) + 1
            else:
                cols1[init:init+seq.shape[1]] = pos + np.where(snpidx)[0] + 1
                cols0[init:init+seq.shape[1]] = (chunk + locindex[iloc] + 1) * -1

        ## fill reference base
        alleles = reftrick(seq, GETCONS)

        ## get the info string column
        tmp0 = np.sum(catg, axis=2)
        tmp1 = tmp0 != 0
        tmp2 = tmp1.sum(axis=1) > 0
        nsamp = np.sum(tmp1, axis=0)
        depth = np.sum(tmp0, axis=0)
        list7 = [["NS={};DP={}".format(i, j)] for i, j in zip(nsamp, depth)]
        if list7:
            cols7[init:init+seq.shape[1]] = list7

        ## default fill cons sites where no variants
        genos[tmp1.T] = "0/0:"

        ## fill cons genotypes for sites with alt alleles for taxa in order
        mask = alleles[:, 1] == 46
        mask += alleles[:, 1] == 45

        obs = alleles[~mask, :]
        alts = seq[:, ~mask]
        who = np.where(mask == False)[0]
        ## fill variable sites
        for site in xrange(alts.shape[1]):
            bases = alts[:, site]
            #LOGGER.info("bases %s", bases)
            ohere = obs[site][obs[site] != 0]
            #LOGGER.info("ohere %s", ohere)
            alls = np.array([DCONS[i] for i in bases], dtype=np.uint32)
            #LOGGER.info("all %s", alls)
            for jdx in xrange(ohere.shape[0]):
                alls[alls == ohere[jdx]] = jdx

            #LOGGER.info("all2 %s", alls)
            ## fill into array
            for cidx in xrange(catg.shape[0]):
                if tmp2[cidx]:
                    if alls[cidx][0] < 5:
                        genos[who[site], cidx] = "/".join(alls[cidx].astype("S1").tolist())+":"
                    else:
                        genos[who[site], cidx] = "./.:"
                    #LOGGER.info("genos filled: %s %s %s", who[site], cidx, genos)

        ## build geno+depth strings
        ## for each taxon enter 4 catg values
        fulltmp = np.zeros((seq.shape[1], catg.shape[0]), dtype="S24")
        for cidx in xrange(catg.shape[0]):
            ## fill catgs from catgs
            tmp0 = [str(i.sum()) for i in catg[cidx]]
            tmp1 = [",".join(i) for i in catg[cidx].astype("S4").tolist()]
            tmp2 = ["".join(i+j+":"+k) for i, j, k in zip(genos[:, cidx], tmp0, tmp1)]
            ## fill tmp allcidx
            fulltmp[:, cidx] = tmp2
        ## write to h5 for this locus
        htmp["vcf"][init:init+seq.shape[1], :] = fulltmp

        cols34[init:init+seq.shape[1], 0] = alleles[:, 0].view("S1")
        cols34[init:init+seq.shape[1], 1] = [",".join([j for j in i if j]) \
                                    for i in alleles[:, 1:].view("S1").tolist()]

        ## advance counter
        init += seq.shape[1]

    ## trim off empty rows if they exist
    withdat = cols0 != 0
    tot = withdat.sum()

    ## get scaffold names
    faidict = {}
    if (data.paramsdict["assembly_method"] in ["reference", "denovo+reference"]) and \
       (os.path.exists(data.paramsdict["reference_sequence"])):
        fai = pd.read_csv(data.paramsdict["reference_sequence"] + ".fai",
                names=['scaffold', 'size', 'sumsize', 'a', 'b'],
                sep="\t")
        faidict = {i+1:j for i,j in enumerate(fai.scaffold)}
    try:
        ## This is hax, but it's the only way it will work. The faidict uses positive numbers
        ## for reference sequence mapped loci for the CHROM/POS info, and it uses negative
        ## numbers for anonymous loci. Both are 1 indexed, which is where that last `+ 2` comes from.
        faidict.update({-i:"locus_{}".format(i-1) for i in xrange(chunk+1, chunk + optim + 2)})
        chroms = [faidict[i] for i in cols0]
    except Exception as inst:
        LOGGER.error("Invalid chromosome dictionary indexwat: {}".format(inst))
        LOGGER.debug("faidict {}".format([str(k)+"/"+str(v) for k, v in faidict.items() if "locus" in v]))
        LOGGER.debug("chroms {}".format([x for x in cols0 if x < 0]))
        raise
    cols0 = np.array(chroms)
    #else:
    #    cols0 = np.array(["locus_{}".format(i) for i in cols0-1])

    ## Only write if there is some data that passed filtering
    if tot:
        LOGGER.debug("Writing data to vcf")
        if not full:
            writer = open(data.outfiles.vcf+".{}".format(chunk), 'w')
        else:
            writer = gzip.open(data.outfiles.vcf+".{}".format(chunk), 'w')

        try:
            ## write in iterations b/c it can be freakin huge.
            ## for cols0 and cols1 the 'newaxis' slice and the transpose
            ## are for turning the 1d arrays into column vectors.
            np.savetxt(writer,
                np.concatenate(
                   (cols0[:tot][np.newaxis].T,
                    cols1[:tot][np.newaxis].T,
                    np.array([["."]]*tot, dtype="S1"),
                    cols34[:tot, :],
                    np.array([["13", "PASS"]]*tot, dtype="S4"),
                    cols7[:tot, :],
                    np.array([["GT:DP:CATG"]]*tot, dtype="S10"),
                    htmp["vcf"][:tot, :],
                    ),
                    axis=1),
                delimiter="\t", fmt="%s")
        except Exception as inst:
            LOGGER.error("Error building vcf file - ".format(inst))
            raise
        writer.close()

    ## close and remove tmp h5
    htmp.close()
    os.remove(tmph)


Instruction:
Write a Python 3 script to
return the most common base at each site in order.

Response:
def reftrick(iseq, consdict):
    """ Returns the most common base at each site in order. """

    altrefs = np.zeros((iseq.shape[1], 4), dtype=np.uint8)
    altrefs[:, 1] = 46

    for col in xrange(iseq.shape[1]):
        ## expand colums with ambigs and remove N-
        fcounts = np.zeros(111, dtype=np.int64)
        counts = np.bincount(iseq[:, col])#, minlength=90)
        fcounts[:counts.shape[0]] = counts
        ## set N and - to zero, wish numba supported minlen arg
        fcounts[78] = 0
        fcounts[45] = 0
        ## add ambig counts to true bases
        for aidx in xrange(consdict.shape[0]):
            nbases = fcounts[consdict[aidx, 0]]
            for _ in xrange(nbases):
                fcounts[consdict[aidx, 1]] += 1
                fcounts[consdict[aidx, 2]] += 1
            fcounts[consdict[aidx, 0]] = 0

        ## now get counts from the modified counts arr
        who = np.argmax(fcounts)
        altrefs[col, 0] = who
        fcounts[who] = 0

        ## if an alt allele fill over the "." placeholder
        who = np.argmax(fcounts)
        if who:
            altrefs[col, 1] = who
            fcounts[who] = 0

            ## if 3rd or 4th alleles observed then add to arr
            who = np.argmax(fcounts)
            altrefs[col, 2] = who
            fcounts[who] = 0

            ## if 3rd or 4th alleles observed then add to arr
            who = np.argmax(fcounts)
            altrefs[col, 3] = who

    return altrefs


Instruction:
Here you have a function in Python 3, explain what it does
def vcfheader(data, names, ofile):
    ## choose reference string
    if data.paramsdict["reference_sequence"]:
        reference = data.paramsdict["reference_sequence"]
    else:
        reference = "pseudo-reference (most common base at site)"

    ##FILTER=<ID=minCov,Description="Data shared across <{mincov} samples">
    ##FILTER=<ID=maxSH,Description="Heterozygosous site shared across >{maxsh} samples">
    header = """\
##fileformat=VCFv4.0
##fileDate={date}
##source=ipyrad_v.{version}
##reference={reference}
##phasing=unphased
##INFO=<ID=NS,Number=1,Type=Integer,Description="Number of Samples With Data">
##INFO=<ID=DP,Number=1,Type=Integer,Description="Total Depth">
##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Read Depth">
##FORMAT=<ID=CATG,Number=1,Type=String,Description="Base Counts (CATG)">
#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t{names}
""".format(date=time.strftime("%Y/%m/%d"),
           version=__version__,
           reference=os.path.basename(reference),
           mincov=data.paramsdict["min_samples_locus"],
           maxsh=data.paramsdict["max_shared_Hs_locus"],
           names="\t".join(names))
    ## WRITE
    ofile.write(header)

Response: Prints header for vcf files


Instruction:
Here you have a function in Python 3, explain what it does
def loci2bpp(name, locifile, imap, guidetree,
    minmap=None,
    maxloci=None,
    infer_sptree=0,
    infer_delimit=0,
    delimit_alg=(0, 5),
    seed=12345,
    burnin=1000,
    nsample=10000,
    sampfreq=2,
    thetaprior=(5, 5),
    tauprior=(4, 2, 1),
    traits_df=None,
    nu=0,
    kappa=0,
    useseqdata=1,
    usetraitdata=1,
    cleandata=0,
    wdir=None,
    finetune=(0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01),
    verbose=0):
    """
    Converts loci file format to bpp file format, i.e., concatenated phylip-like
    format, and produces imap and ctl input files for bpp.

    Parameters:
    -----------
    name:
        A prefix name for output files that will be produced
    locifile:
        A .loci file produced by ipyrad.
    imap:
        A Python dictionary with 'species' names as keys, and lists of sample
        names for the values. Any sample that is not included in the imap
        dictionary will be filtered out of the data when converting the .loci
        file into the bpp formatted sequence file. Each species in the imap
        dictionary must also be present in the input 'guidetree'.
    guidetree:
        A newick string species tree hypothesis [e.g., (((a,b),(c,d)),e);]
        All species in the imap dictionary must also be present in the guidetree

    Optional parameters:
    --------------------
    infer_sptree:
        Default=0, only infer parameters on a fixed species tree. If 1, then the
        input tree is treated as a guidetree and tree search is employed to find
        the best tree. The results will include support values for the inferred
        topology.
    infer_delimit:
        Default=0, no delimitation. If 1 then splits in the tree that separate
        'species' will be collapsed to test whether fewer species are a better
        fit to the data than the number in the input guidetree.
    delimit_alg:
        Species delimitation algorithm. This is a tuple. The first value
        is the algorithm (0 or 1) and the following values are arguments
        for the given algorithm. See other ctl files for examples of what the
        delimitation line looks like. This is where you can enter the params
        (e.g., alpha, migration) for the two different algorithms.
        For example, the following args would produce the following ctl lines:
           alg=0, epsilon=5
           > delimit_alg = (0, 5)
           speciesdelimitation = 1 0 5

           alg=1, alpha=2, migration=1
           > delimit_alg = (1, 2, 1)
           speciesdelimitation = 1 1 2 1

           alg=1, alpha=2, migration=1, diagnosis=0, ?=1
           > delimit_alg = (1, 2, 1, 0, 1)
           speciesdelimitation = 1 1 2 1 0 1
    seed:
        A random number seed at start of analysis.
    burnin:
        Number of burnin generations in mcmc
    nsample:
        Number of mcmc generations to run.
    sampfreq:
        How often to sample from the mcmc chain.
    thetaprior:
        Prior on theta (4Neu), gamma distributed. mean = a/b. e.g., (5, 5)
    tauprior
        Prior on root tau, gamma distributed mean = a/b. Last number is
        dirichlet prior for other taus. e.g., (4, 2, 1)
    traits_df:
        A pandas DataFrame with trait data properly formatted. This means only
        quantitative traits are included, and missing values are NaN.
        The first column contains sample names, with "Indiv" as the header.
        The following columns have a header row with trait names. This script
        will write a CSV trait file with trait values mean-standardized, with
        NaN replaced by "NA", and with sample not present in IMAP removed.
    nu:
        A prior on phenotypic trait variance (0) for iBPP analysis.
    kappa:
        A prior on phenotypic trait mean (0) for iBPP analysis.
    useseqdata:
        If false inference proceeds without sequence data (can be used to test
        the effect of priors on the tree distributions).
    usetraitdata:
        If false inference proceeds without trait data (can be used to test
        the effect of priors on the trait distributions).
    cleandata:
        If 1 then sites with missing or hetero characters are removed.
    wdir:
        A working directory to write files to.
    finetune:
        See bpp documentation.
    verbose:
        If verbose=1 the ctl file text will also be written to screen (stderr).

    """
    ## check args
    if not imap:
        raise IPyradWarningExit(IMAP_REQUIRED)
    if minmap:
        if minmap.keys() != imap.keys():
            raise IPyradWarningExit(KEYS_DIFFER)

    ## working directory, make sure it exists
    if wdir:
        wdir = os.path.abspath(wdir)
        if not os.path.exists(wdir):
            raise IPyradWarningExit(" working directory (wdir) does not exist")
    else:
        wdir = os.path.curdir

    ## if traits_df then we make '.ibpp' files
    prog = 'bpp'
    if isinstance(traits_df, pd.DataFrame):
        prog = 'ibpp'
    outfile = OPJ(wdir, "{}.{}.seq.txt".format(name, prog))
    mapfile = OPJ(wdir, "{}.{}.imap.txt".format(name, prog))

    ## open outhandles
    fout = open(outfile, 'w')
    fmap = open(mapfile, 'w')

    ## parse the loci file
    with open(locifile, 'r') as infile:
        ## split on "//" for legacy compatibility
        loci = infile.read().strip().split("|\n")
        nloci = len(loci)

    ## all samples
    samples = list(itertools.chain(*imap.values()))

    ## iterate over loci, printing to outfile
    nkept = 0
    for iloc in xrange(nloci):
        lines = loci[iloc].split("//")[0].split()
        names = lines[::2]
        names = ["^"+i for i in names]
        seqs = [list(i) for i in lines[1::2]]
        seqlen = len(seqs[0])

        ## whether to skip this locus based on filters below
        skip = 0

        ## if minmap filter for sample coverage
        if minmap:
            covd = {}
            for group, vals in imap.items():
                covd[group] = sum(["^"+i in names for i in vals])
            ## check that coverage is good enough
            if not all([covd[group] >= minmap[group] for group in minmap]):
                skip = 1

        ## too many loci?
        if maxloci:
            if nkept >= maxloci:
                skip = 1

        ## build locus as a string
        if not skip:
            ## convert to phylip with caret starter and replace - with N.
            data = ["{:<30} {}".format(i, "".join(k).replace("-", "N")) for \
                (i, k) in zip(names, seqs) if i[1:] in samples]

            ## if not empty, write to the file
            if data:
                fout.write("{} {}\n\n{}\n\n"\
                           .format(len(data), seqlen, "\n".join(data)))
                nkept += 1

    ## close up shop
    fout.close()

    ## write the imap file:
    data = ["{:<30} {}".format(val, key) for key \
            in sorted(imap) for val in imap[key]]
    fmap.write("\n".join(data))
    fmap.close()

    ## write ctl file
    write_ctl(name, imap, guidetree, nkept,
              infer_sptree, infer_delimit, delimit_alg,
              seed, burnin, nsample, sampfreq,
              thetaprior, tauprior, traits_df, nu, kappa,
              cleandata, useseqdata, usetraitdata, wdir,
              finetune, verbose)

    ## print message?
    sys.stderr.write("new files created ({} loci, {} species, {} samples)\n"\
                     .format(nkept, len(imap.keys()),
                             sum([len(i) for i in imap.values()])))
    sys.stderr.write("  {}.{}.seq.txt\n".format(name, prog))
    sys.stderr.write("  {}.{}.imap.txt\n".format(name, prog))
    sys.stderr.write("  {}.{}.ctl.txt\n".format(name, prog))
    if isinstance(traits_df, pd.DataFrame):
        sys.stderr.write("  {}.{}.traits.txt\n".format(name, prog))

    ## return the ctl file string
    return os.path.abspath(
        "{}.{}.ctl.txt".format(OPJ(wdir, name), prog))

Response: This function takes a loci file and returns a bpp file format for the given loci file.


Instruction:
How would you code a function in Python 3 to
write a control file

Response:
def write_ctl(name, imap, guidetree, nloci,
              infer_sptree, infer_delimit, delimit_alg,
              seed, burnin, nsample, sampfreq,
              thetaprior, tauprior, traits_df, nu0, kappa0,
              cleandata, useseqdata, usetraitdata, wdir,
              finetune, verbose):

    """ write outfile with any args in argdict """

    ## A string to store ctl info
    ctl = []

    ## check the tree (can do this better once we install ete3 w/ ipyrad)
    if not guidetree.endswith(";"):
        guidetree += ";"

    ## if traits_df then we make '.ibpp' files
    prog = 'bpp'
    if isinstance(traits_df, pd.DataFrame):
        prog = 'ibpp'

    ## write the top header info
    ctl.append("seed = {}".format(seed))
    ctl.append("seqfile = {}.{}.seq.txt".format(OPJ(wdir, name), prog))
    ctl.append("Imapfile = {}.{}.imap.txt".format(OPJ(wdir, name), prog))
    ctl.append("mcmcfile = {}.{}.mcmc.txt".format(OPJ(wdir, name), prog))
    ctl.append("outfile = {}.{}.out.txt".format(OPJ(wdir, name), prog))
    if isinstance(traits_df, pd.DataFrame):
        ctl.append("traitfile = {}.{}.traits.txt".format(OPJ(wdir, name), prog))

    ## number of loci (checks that seq file exists and parses from there)
    ctl.append("nloci = {}".format(nloci))
    ctl.append("usedata = {}".format(useseqdata))
    ctl.append("cleandata = {}".format(cleandata))

    ## infer species tree
    if infer_sptree:
        ctl.append("speciestree = 1 0.4 0.2 0.1")
    else:
        ctl.append("speciestree = 0")

    ## infer delimitation (with algorithm 1 by default)
    ctl.append("speciesdelimitation = {} {} {}"\
               .format(infer_delimit, delimit_alg[0],
                       " ".join([str(i) for i in delimit_alg[1:]])))

    ## if using iBPP (if not traits_df, we assume you're using bpp (v.3.3+)
    if isinstance(traits_df, pd.DataFrame):
        ## check that the data frame is properly formatted
        try:
            traits_df.values.astype(float)
        except Exception:
            raise IPyradWarningExit(PDREAD_ERROR)

        ## subsample to keep only samples that are in IMAP, we do not need to
        ## standarize traits b/c ibpp does that for us.
        samples = sorted(list(itertools.chain(*imap.values())))
        didx = [list(traits_df.index).index(i) for i in traits_df.index \
                if i not in samples]
        dtraits = traits_df.drop(traits_df.index[didx])

        ## mean standardize traits values after excluding samples
        straits = dtraits.apply(lambda x: (x - x.mean()) / (x.std()))

        ## convert NaN to "NA" cuz that's what ibpp likes, and write to file
        ftraits = straits.fillna("NA")
        traitdict = ftraits.T.to_dict("list")

        ## get reverse imap dict
        rev = {val:key for key in sorted(imap) for val in imap[key]}

        ## write trait file
        traitfile = "{}.{}.traits.txt".format(os.path.join(wdir, name), prog)
        with open(traitfile, 'w') as tout:
            tout.write("Indiv\n")
            tout.write("\t".join(
                ['Species'] + list(ftraits.columns))+"\n"
                )
            #for key in sorted(traitdict):
            #    tout.write("\t".join([key, rev[key]] + \
            #        ["^"+str(i) for i in traitdict[key]])+"\n"
            #        )
            nindT = 0
            for ikey in sorted(imap.keys()):
                samps = imap[ikey]
                for samp in sorted(samps):
                    if samp in traitdict:
                        tout.write("\t".join([samp, rev[samp]] + \
                            [str(i) for i in traitdict[samp]])+"\n"
                        )
                        nindT += 1

        #    tout.write("Indiv\n"+"\t".join(["Species"]+\
        #    ["t_{}".format(i) for i in range(len(traitdict.values()[0]))])+"\n")
        #    for key in sorted(traitdict):
        #        print >>tout, "\t".join([key, rev[key]] + \
        #                                [str(i) for i in traitdict[key]])
        #ftraits.to_csv(traitfile)

        ## write ntraits and nindT and traitfilename
        ctl.append("ntraits = {}".format(traits_df.shape[1]))
        ctl.append("nindT = {}".format(nindT))  #traits_df.shape[0]))
        ctl.append("usetraitdata = {}".format(usetraitdata))
        ctl.append("useseqdata = {}".format(useseqdata))

        ## trait priors
        ctl.append("nu0 = {}".format(nu0))
        ctl.append("kappa0 = {}".format(kappa0))

        ## remove ibpp incompatible options
        ctl.remove("usedata = {}".format(useseqdata))
        ctl.remove("speciestree = {}".format(infer_sptree))

    ## get tree values
    nspecies = str(len(imap))
    species = " ".join(sorted(imap))
    ninds = " ".join([str(len(imap[i])) for i in sorted(imap)])

    ## write the tree
    ctl.append("""\
species&tree = {} {}
                 {}
                 {}""".format(nspecies, species, ninds, guidetree))

    ## priors
    ctl.append("thetaprior = {} {}".format(*thetaprior))
    ctl.append("tauprior = {} {} {}".format(*tauprior))

    ## other values, fixed for now
    ctl.append("finetune = 1: {}".format(" ".join([str(i) for i in finetune])))
    #CTL.append("finetune = 1: 1 0.002 0.01 0.01 0.02 0.005 1.0")
    ctl.append("print = 1 0 0 0")
    ctl.append("burnin = {}".format(burnin))
    ctl.append("sampfreq = {}".format(sampfreq))
    ctl.append("nsample = {}".format(nsample))

    ## write out the ctl file
    with open("{}.{}.ctl.txt".format(OPJ(wdir, name), prog), 'w') as out:
        out.write("\n".join(ctl))

    ## if verbose print ctl
    if verbose:
        sys.stderr.write("ctl file\n--------\n"+"\n".join(ctl)+"\n--------\n\n")


Instruction:
Create a Python 3 function for
collapsing outgroup in ete Tree for easier viewing

Response:
def _collapse_outgroup(tree, taxdicts):
    """ collapse outgroup in ete Tree for easier viewing """
    ## check that all tests have the same outgroup
    outg = taxdicts[0]["p4"]
    if not all([i["p4"] == outg for i in taxdicts]):
        raise Exception("no good")

    ## prune tree, keep only one sample from outgroup
    tre = ete.Tree(tree.write(format=1)) #tree.copy(method="deepcopy")
    alltax = [i for i in tre.get_leaf_names() if i not in outg]
    alltax += [outg[0]]
    tre.prune(alltax)
    tre.search_nodes(name=outg[0])[0].name = "outgroup"
    tre.ladderize()

    ## remove other ougroups from taxdicts
    taxd = copy.deepcopy(taxdicts)
    newtaxdicts = []
    for test in taxd:
        #test["p4"] = [outg[0]]
        test["p4"] = ["outgroup"]
        newtaxdicts.append(test)

    return tre, newtaxdicts


Instruction:
Given the following Python 3 function, write the documentation
def _decompose_tree(ttree, orient='right', use_edge_lengths=True):

    ## set attributes
    ttree._orient = orient
    ttree._use_edge_lengths = use_edge_lengths
    ult = use_edge_lengths == False

    ## map numeric values to internal nodes from root to tips
    names = {}
    idx = 0
    for node in ttree.tree.traverse("preorder"):
        if not node.is_leaf():
            if node.name:
                names[idx] = node.name
            else:
                names[idx] = idx
                node.name = str(idx)
            node.idx = idx
            idx += 1

    ## map number to the tips, these will be the highest numbers
    for node in ttree.tree.get_leaves():
        names[idx] = node.name
        node.idx = idx
        idx += 1

    ## create empty edges and coords arrays
    ttree.node_labels = names
    ttree.tip_labels = ttree.tree.get_leaf_names()
    #self.tip_labels = self.tree.get_leaf_names()[::-1]
    #self.node_labels = self.names
    ttree.edges = np.zeros((idx - 1, 2), dtype=int)
    ttree.verts = np.zeros((idx, 2), dtype=float)
    ttree._lines = []        # np.zeros((ntips-1), dtype=int)
    ttree._coords = []       # np.zeros((idx * 2 - ntips), dtype=float)

    ## postorder: first children and then parents. This moves up the list .
    nidx = 0
    tip_num = len(ttree.tree.get_leaves()) - 1

    ## tips to root to fill in the verts and edges
    for node in ttree.tree.traverse("postorder"):
        if node.is_leaf():
            ## set the xy-axis positions of the tips
            node.y = ttree.tree.get_distance(node)
            if ult:
                node.y = 0.
            node.x = tip_num
            tip_num -= 1

            ## edges connect this vert to
            ttree.verts[node.idx] = [node.x, node.y]
            ttree.edges[nidx] = [node.up.idx, node.idx]

        elif node.is_root():
            node.y = ttree.tree.get_distance(node)
            if ult:
                node.y = -1 * node.get_farthest_leaf(True)[1] - 1
            node.x = sum(i.x for i in node.children) / float(len(node.children))
            ttree.verts[node.idx] = [node.x, node.y]

        else:
            ## create new nodes left and right
            node.y = ttree.tree.get_distance(node)
            if ult:
                node.y = -1 * node.get_farthest_leaf(True)[1] - 1
            node.x = sum(i.x for i in node.children) / float(len(node.children))
            ttree.edges[nidx, :] = [node.up.idx, node.idx]
            ttree.verts[node.idx] = [node.x, node.y]
        nidx += 1

    ## root to tips to fill in the coords and lines
    cidx = 0
    for node in ttree.tree.traverse():
        ## add yourself
        if not node.is_leaf():
            ttree._coords += [[node.x, node.y]]
            pidx = cidx
            cidx += 1
            for child in node.children:
                ## add children
                ttree._coords += [[child.x, node.y], [child.x, child.y]]
                ttree._lines += [[pidx, cidx]]    ## connect yourself to newx
                ttree._lines += [[cidx, cidx+1]]  ## connect newx to child
                cidx += 2
    ttree._coords = np.array(ttree._coords, dtype=float)
    ttree._lines = np.array(ttree._lines, dtype=int)

    ## invert for sideways trees
    if ttree._orient in ['up', 0]:
        pass
    if ttree._orient in ['left', 1]:
        ttree.verts[:, 1] = ttree.verts[:, 1] * -1
        ttree.verts = ttree.verts[:, [1, 0]]
        ttree._coords[:, 1] = ttree._coords[:, 1] * -1
        ttree._coords = ttree._coords[:, [1, 0]]
    if ttree._orient in ['down', 0]:
        ttree.verts[:, 1] = ttree.verts[:, 1] * -1
        ttree._coords[:, 1] = ttree._coords[:, 1] * -1
    if ttree._orient in ['right', 3]:
        ttree.verts = ttree.verts[:, [1, 0]]
        ttree._coords = ttree._coords[:, [1, 0]]

Response: decompose a tree into a single tree structure


Instruction:
Implement a Python 3 function for
drawing the tree using toyplot. graph. MacroMacro

Response:
def draw(
        self,
        show_tip_labels=True,
        show_node_support=False,
        use_edge_lengths=False,
        orient="right",
        print_args=False,
        *args,
        **kwargs):
        """
        plot the tree using toyplot.graph.

        Parameters:
        -----------
            show_tip_labels: bool
                Show tip names from tree.
            use_edge_lengths: bool
                Use edge lengths from newick tree.
            show_node_support: bool
                Show support values at nodes using a set of default
                options.

            ...
        """
        ## re-decompose tree for new orient and edges args
        self._decompose_tree(orient=orient, use_edge_lengths=use_edge_lengths)

        ## update kwargs with entered args and all other kwargs
        dwargs = {}
        dwargs["show_tip_labels"] = show_tip_labels
        dwargs["show_node_support"] = show_node_support
        dwargs.update(kwargs)

        ## pass to panel plotter
        canvas, axes, panel = tree_panel_plot(self, print_args, **dwargs)
        return canvas, axes, panel


Instruction:
Given the following Python 3 function, write the documentation
def tree_panel_plot(ttree,
    print_args=False,
    *args,
    **kwargs):
    """
    signature...
    """

    ## create Panel plot object and set height & width
    panel = Panel(ttree)          #tree, edges, verts, names)
    if not kwargs.get("width"):
        panel.kwargs["width"] = min(1000, 25*len(panel.tree))
    if not kwargs.get("height"):
        panel.kwargs["height"] = panel.kwargs["width"]

    ## update defaults with kwargs & update size based on ntips & ntests
    panel.kwargs.update(kwargs)

    ## magic node label arguments overrides others
    if panel.kwargs["show_node_support"]:
        nnodes = sum(1 for i in panel.tree.traverse()) - len(panel.tree)
        ## set node values
        supps = [int(panel.tree.search_nodes(idx=j)[0].support) \
                 for j in range(nnodes)]
        if not panel.kwargs["vsize"]:
            panel.kwargs["vsize"] = 20
        sizes = [panel.kwargs["vsize"] for j in range(nnodes)]
        ## add leaf values
        supps += [""] * len(panel.tree)
        sizes += [0] * len(panel.tree)
        ## override args
        panel.kwargs["vlabel"] = supps
        panel.kwargs["vsize"] = sizes
        panel.kwargs["vlshow"] = True
        #panel.kwargs["vmarker"] = 's'  ## square
        ## if unrooted then hide root node scores
        if len(panel.tree.children) > 2:
            supps[0] = ""
            sizes[0] = 0
        #print(panel.kwargs["vlabels"])
        #print(panel.kwargs["vsize"])
    elif panel.kwargs.get("vlabel"):
        panel.kwargs["vlabel"] = panel.kwargs["vlabel"]
        panel.kwargs["vlshow"] = True
    else:
        panel.kwargs["vlabel"] = panel.node_labels.keys() #names.keys()

    ## debugger / see all options
    if print_args:
        print(panel.kwargs)

    ## maybe add panels for plotting tip traits in the future
    ## ...

    ## create a canvas and a single cartesian coord system
    canvas = toyplot.Canvas(height=panel.kwargs['height'], width=panel.kwargs['width'])
    axes = canvas.cartesian(bounds=("10%", "90%", "10%", "90%"))
    axes.show = panel.kwargs["show_axes"]

    ## add panel plots to the axis
    panel._panel_tree(axes)
    if panel.kwargs["show_tip_labels"]:
        panel._panel_tip_labels(axes)

    return canvas, axes, panel

Response: create a Panel object and plot it


Instruction:
How would you explain what the following Python 3 function does
def get_quick_depths(data, sample):

    ## use existing sample cluster path if it exists, since this
    ## func can be used in step 4 and that can occur after merging
    ## assemblies after step3, and if we then referenced by data.dirs.clusts
    ## the path would be broken.
    ##
    ## If branching at step 3 to test different clust thresholds, the
    ## branched samples will retain the samples.files.clusters of the
    ## parent (which have the clust_threshold value of the parent), so
    ## it will look like nothing has changed. If we call this func
    ## from step 3 then it indicates we are in a branch and should
    ## reset the sample.files.clusters handle to point to the correct
    ## data.dirs.clusts directory. See issue #229.
    ## Easier to just always trust that samples.files.clusters is right,
    ## no matter what step?
    #if sample.files.clusters and not sample.stats.state == 3:
    #    pass
    #else:
    #    ## set cluster file handles
    sample.files.clusters = os.path.join(
         data.dirs.clusts, sample.name+".clustS.gz")

    ## get new clustered loci
    fclust = data.samples[sample.name].files.clusters
    clusters = gzip.open(fclust, 'r')
    pairdealer = itertools.izip(*[iter(clusters)]*2)

    ## storage
    depths = []
    maxlen = []

    ## start with cluster 0
    tdepth = 0
    tlen = 0

    ## iterate until empty
    while 1:
        ## grab next
        try:
            name, seq = pairdealer.next()
        except StopIteration:
            break

        ## if not the end of a cluster
        #print name.strip(), seq.strip()
        if name.strip() == seq.strip():
            depths.append(tdepth)
            maxlen.append(tlen)
            tlen = 0
            tdepth = 0

        else:
            tdepth += int(name.split(";")[-2][5:])
            tlen = len(seq)

    ## return
    clusters.close()
    return np.array(maxlen), np.array(depths)

Response: get the quick depths of the clustered loci


Instruction:
Can you generate a brief explanation for the following Python 3 code
def sample_cleanup(data, sample):

    ## get maxlen and depths array from clusters
    maxlens, depths = get_quick_depths(data, sample)

    try:
        depths.max()
    except ValueError:
        ## If depths is an empty array max() will raise
        print("    no clusters found for {}".format(sample.name))
        return

    ## Test if depths is non-empty, but just full of zeros.
    if depths.max():
        ## store which min was used to calculate hidepth here
        sample.stats_dfs.s3["hidepth_min"] = data.paramsdict["mindepth_majrule"]

        ## If our longest sequence is longer than the current max_fragment_length
        ## then update max_fragment_length. For assurance we require that
        ## max len is 4 greater than maxlen, to allow for pair separators.
        hidepths = depths >= data.paramsdict["mindepth_majrule"]
        maxlens = maxlens[hidepths]

        ## Handle the case where there are no hidepth clusters
        if maxlens.any():
            maxlen = int(maxlens.mean() + (2.*maxlens.std()))
        else:
            maxlen = 0
        if maxlen > data._hackersonly["max_fragment_length"]:
            data._hackersonly["max_fragment_length"] = maxlen + 4

        ## make sense of stats
        keepmj = depths[depths >= data.paramsdict["mindepth_majrule"]]
        keepstat = depths[depths >= data.paramsdict["mindepth_statistical"]]

        ## sample summary stat assignments
        sample.stats["state"] = 3
        sample.stats["clusters_total"] = depths.shape[0]
        sample.stats["clusters_hidepth"] = keepmj.shape[0]

        ## store depths histogram as a dict. Limit to first 25 bins
        bars, bins = np.histogram(depths, bins=range(1, 26))
        sample.depths = {int(i):v for i, v in zip(bins, bars) if v}

        ## sample stat assignments
        ## Trap numpy warnings ("mean of empty slice") printed by samples
        ## with few reads.
        with warnings.catch_warnings():
            warnings.simplefilter("ignore", category=RuntimeWarning)
            sample.stats_dfs.s3["merged_pairs"] = sample.stats.reads_merged
            sample.stats_dfs.s3["clusters_total"] = depths.shape[0]
            try:
                sample.stats_dfs.s3["clusters_hidepth"] = int(sample.stats["clusters_hidepth"])
            except ValueError:
                ## Handle clusters_hidepth == NaN
                sample.stats_dfs.s3["clusters_hidepth"] = 0
            sample.stats_dfs.s3["avg_depth_total"] = depths.mean()
            LOGGER.debug("total depth {}".format(sample.stats_dfs.s3["avg_depth_total"]))
            sample.stats_dfs.s3["avg_depth_mj"] = keepmj.mean()
            LOGGER.debug("mj depth {}".format(sample.stats_dfs.s3["avg_depth_mj"]))
            sample.stats_dfs.s3["avg_depth_stat"] = keepstat.mean()
            sample.stats_dfs.s3["sd_depth_total"] = depths.std()
            sample.stats_dfs.s3["sd_depth_mj"] = keepmj.std()
            sample.stats_dfs.s3["sd_depth_stat"] = keepstat.std()

    else:
        print("    no clusters found for {}".format(sample.name))

    ## Get some stats from the bam files
    ## This is moderately hackish. samtools flagstat returns
    ## the number of reads in the bam file as the first element
    ## of the first line, this call makes this assumption.
    if not data.paramsdict["assembly_method"] == "denovo":
        refmap_stats(data, sample)

    log_level = logging.getLevelName(LOGGER.getEffectiveLevel())

    if not log_level == "DEBUG":
        ## Clean up loose files only if not in DEBUG
        ##- edits/*derep, utemp, *utemp.sort, *htemp, *clust.gz
        derepfile = os.path.join(data.dirs.edits, sample.name+"_derep.fastq")
        mergefile = os.path.join(data.dirs.edits, sample.name+"_merged_.fastq")
        uhandle = os.path.join(data.dirs.clusts, sample.name+".utemp")
        usort = os.path.join(data.dirs.clusts, sample.name+".utemp.sort")
        hhandle = os.path.join(data.dirs.clusts, sample.name+".htemp")
        clusters = os.path.join(data.dirs.clusts, sample.name+".clust.gz")

        for f in [derepfile, mergefile, uhandle, usort, hhandle, clusters]:
            try:
                os.remove(f)
            except:
                pass

Response: This function is used to clean up the sample and store the stats and cleanup and link to samples.


Instruction:
Write a Python 3 function for
aligning the read1s and read2s in the file with muscle.

Response:
def persistent_popen_align3(clusts, maxseqs=200, is_gbs=False):
    """ keeps a persistent bash shell open and feeds it muscle alignments """

    ## create a separate shell for running muscle in, this is much faster
    ## than spawning a separate subprocess for each muscle call
    proc = sps.Popen(["bash"],
                     stdin=sps.PIPE,
                     stdout=sps.PIPE,
                     universal_newlines=True)

    ## iterate over clusters in this file until finished
    aligned = []
    for clust in clusts:

        ## new alignment string for read1s and read2s
        align1 = ""
        align2 = ""

        ## don't bother aligning if only one seq
        if clust.count(">") == 1:
            aligned.append(clust.replace(">", "").strip())
        else:

            ## do we need to split the alignment? (is there a PE insert?)
            try:
                ## make into list (only read maxseqs lines, 2X cuz names)
                lclust = clust.split()[:maxseqs*2]

                ## try to split cluster list at nnnn separator for each read
                lclust1 = list(itertools.chain(*zip(\
                     lclust[::2], [i.split("nnnn")[0] for i in lclust[1::2]])))
                lclust2 = list(itertools.chain(*zip(\
                     lclust[::2], [i.split("nnnn")[1] for i in lclust[1::2]])))

                ## put back into strings
                clust1 = "\n".join(lclust1)
                clust2 = "\n".join(lclust2)

                ## Align the first reads.
                ## The muscle command with alignment as stdin and // as splitter
                cmd1 = "echo -e '{}' | {} -quiet -in - ; echo {}"\
                        .format(clust1, ipyrad.bins.muscle, "//")

                ## send cmd1 to the bash shell
                print(cmd1, file=proc.stdin)

                ## read the stdout by line until splitter is reached
                ## meaning that the alignment is finished.
                for line in iter(proc.stdout.readline, '//\n'):
                    align1 += line

                ## Align the second reads.
                ## The muscle command with alignment as stdin and // as splitter
                cmd2 = "echo -e '{}' | {} -quiet -in - ; echo {}"\
                        .format(clust2, ipyrad.bins.muscle, "//")

                ## send cmd2 to the bash shell
                print(cmd2, file=proc.stdin)

                ## read the stdout by line until splitter is reached
                ## meaning that the alignment is finished.
                for line in iter(proc.stdout.readline, '//\n'):
                    align2 += line

                ## join up aligned read1 and read2 and ensure names order matches
                la1 = align1[1:].split("\n>")
                la2 = align2[1:].split("\n>")
                dalign1 = dict([i.split("\n", 1) for i in la1])
                dalign2 = dict([i.split("\n", 1) for i in la2])
                align1 = []
                try:
                    keys = sorted(dalign1.keys(), key=DEREP, reverse=True)
                except ValueError as inst:
                    ## Lines is empty. This means the call to muscle alignment failed.
                    ## Not sure how to handle this, but it happens only very rarely.
                    LOGGER.error("Muscle alignment failed: Bad clust - {}\nBad lines - {}"\
                                .format(clust, lines))
                    continue

                ## put seed at top of alignment
                seed = [i for i in keys if i.split(";")[-1][0]=="*"][0]
                keys.pop(keys.index(seed))
                keys = [seed] + keys
                for key in keys:
                    align1.append("\n".join([key,
                                    dalign1[key].replace("\n", "")+"nnnn"+\
                                    dalign2[key].replace("\n", "")]))

                ## append aligned cluster string
                aligned.append("\n".join(align1).strip())

            ## Malformed clust. Dictionary creation with only 1 element will raise.
            except ValueError as inst:
                LOGGER.debug("Bad PE cluster - {}\nla1 - {}\nla2 - {}".format(\
                                clust, la1, la2))

            ## Either reads are SE, or at least some pairs are merged.
            except IndexError:

                ## limit the number of input seqs
                lclust = "\n".join(clust.split()[:maxseqs*2])

                ## the muscle command with alignment as stdin and // as splitter
                cmd = "echo -e '{}' | {} -quiet -in - ; echo {}"\
                            .format(lclust, ipyrad.bins.muscle, "//")

                ## send cmd to the bash shell (TODO: PIPE could overflow here!)
                print(cmd, file=proc.stdin)

                ## read the stdout by line until // is reached. This BLOCKS.
                for line in iter(proc.stdout.readline, '//\n'):
                    align1 += line

                ## remove '>' from names, and '\n' from inside long seqs
                lines = align1[1:].split("\n>")

                try:
                    ## find seed of the cluster and put it on top.
                    seed = [i for i in lines if i.split(";")[-1][0]=="*"][0]
                    lines.pop(lines.index(seed))
                    lines = [seed] + sorted(lines, key=DEREP, reverse=True)
                except ValueError as inst:
                    ## Lines is empty. This means the call to muscle alignment failed.
                    ## Not sure how to handle this, but it happens only very rarely.
                    LOGGER.error("Muscle alignment failed: Bad clust - {}\nBad lines - {}"\
                                .format(clust, lines))
                    continue

                ## format remove extra newlines from muscle
                aa = [i.split("\n", 1) for i in lines]
                align1 = [i[0]+'\n'+"".join([j.replace("\n", "") for j in i[1:]]) for i in aa]

                ## trim edges in sloppy gbs/ezrad data. Maybe relevant to other types too...
                if is_gbs:
                    align1 = gbs_trim(align1)

                ## append to aligned
                aligned.append("\n".join(align1).strip())

    # cleanup
    proc.stdout.close()
    if proc.stderr:
        proc.stderr.close()
    proc.stdin.close()
    proc.wait()

    ## return the aligned clusters
    return aligned


Instruction:
Can you generate the documentation for the following Python 3 function
def gbs_trim(align1):
    leftmost = rightmost = None
    dd = {k:v for k,v in [j.rsplit("\n", 1) for j in align1]}
    seed = [i for i in dd.keys() if i.rsplit(";")[-1][0] == "*"][0]
    leftmost = [i != "-" for i in dd[seed]].index(True)
    revs = [i for i in dd.keys() if i.rsplit(";")[-1][0] == "-"]
    if revs:
        subright = max([[i!="-" for i in seq[::-1]].index(True) \
            for seq in [dd[i] for i in revs]])
    else:
        subright = 0
    rightmost = len(dd[seed]) - subright

    ## if locus got clobbered then print place-holder NNN
    names, seqs = zip(*[i.rsplit("\n", 1) for i in align1])
    if rightmost > leftmost:
        newalign1 = [n+"\n"+i[leftmost:rightmost] for i,n in zip(seqs, names)]
    else:
        newalign1 = [n+"\nNNN" for i,n in zip(seqs, names)]
    return newalign1

Response: trims the input sequence of read sequences


Instruction:
How would you code a function in Python 3 to
align the file with the given handle and parses the data into a single file.

Response:
def align_and_parse(handle, max_internal_indels=5, is_gbs=False):
    """ much faster implementation for aligning chunks """

    ## data are already chunked, read in the whole thing. bail if no data.
    try:
        with open(handle, 'rb') as infile:
            clusts = infile.read().split("//\n//\n")
            ## remove any empty spots
            clusts = [i for i in clusts if i]
            ## Skip entirely empty chunks
            if not clusts:
                raise IPyradError
    except (IOError, IPyradError):
        LOGGER.debug("skipping empty chunk - {}".format(handle))
        return 0

    ## count discarded clusters for printing to stats later
    highindels = 0

    ## iterate over clusters sending each to muscle, splits and aligns pairs
    try:
        aligned = persistent_popen_align3(clusts, 200, is_gbs)
    except Exception as inst:
        LOGGER.debug("Error in handle - {} - {}".format(handle, inst))
        #raise IPyradWarningExit("error hrere {}".format(inst))
        aligned = []

    ## store good alignments to be written to file
    refined = []

    ## filter and trim alignments
    for clust in aligned:

        ## check for too many internal indels
        filtered = aligned_indel_filter(clust, max_internal_indels)

        ## reverse complement matches. No longer implemented.
        #filtered = overshoot_filter(clust)

        ## finally, add to outstack if alignment is good
        if not filtered:
            refined.append(clust)#"\n".join(stack))
        else:
            highindels += 1

    ## write to file after
    if refined:
        outhandle = handle.rsplit(".", 1)[0]+".aligned"
        with open(outhandle, 'wb') as outfile:
            outfile.write("\n//\n//\n".join(refined)+"\n")

    ## remove the old tmp file
    log_level = logging.getLevelName(LOGGER.getEffectiveLevel())
    if not log_level == "DEBUG":
        os.remove(handle)
    return highindels


Instruction:
Can you generate the documentation for the following Python 3 function
def aligned_indel_filter(clust, max_internal_indels):

    ## make into list
    lclust = clust.split()

    ## paired or not
    try:
        seq1 = [i.split("nnnn")[0] for i in lclust[1::2]]
        seq2 = [i.split("nnnn")[1] for i in lclust[1::2]]
        intindels1 = [i.rstrip("-").lstrip("-").count("-") for i in seq1]
        intindels2 = [i.rstrip("-").lstrip("-").count("-") for i in seq2]
        intindels = intindels1 + intindels2
        if max(intindels) > max_internal_indels:
            return 1

    except IndexError:
        seq1 = lclust[1::2]
        intindels = [i.rstrip("-").lstrip("-").count("-") for i in seq1]
        if max(intindels) > max_internal_indels:
            return 1

    return 0

Response: checks for too many internal indels in muscle aligned clusters