Instruction:
Implement a Python 3 function for
creating cluster files for the unaligned reads of the sample.

Response:
def build_clusters(data, sample, maxindels):
    """
    Combines information from .utemp and .htemp files to create .clust files,
    which contain un-aligned clusters. Hits to seeds are only kept in the
    cluster if the number of internal indels is less than 'maxindels'.
    By default, we set maxindels=6 for this step (within-sample clustering).
    """

    ## If reference assembly then here we're clustering the unmapped reads
    if "reference" in data.paramsdict["assembly_method"]:
        derepfile = os.path.join(data.dirs.edits, sample.name+"-refmap_derep.fastq")
    else:
        derepfile = os.path.join(data.dirs.edits, sample.name+"_derep.fastq")
    ## i/o vsearch files
    uhandle = os.path.join(data.dirs.clusts, sample.name+".utemp")
    usort = os.path.join(data.dirs.clusts, sample.name+".utemp.sort")
    hhandle = os.path.join(data.dirs.clusts, sample.name+".htemp")

    ## create an output file to write clusters to
    sample.files.clusters = os.path.join(data.dirs.clusts, sample.name+".clust.gz")
    clustsout = gzip.open(sample.files.clusters, 'wb')

    ## Sort the uhandle file so we can read through matches efficiently
    cmd = ["sort", "-k", "2", uhandle, "-o", usort]
    proc = sps.Popen(cmd, close_fds=True)
    _ = proc.communicate()[0]

    ## load ALL derep reads into a dictionary (this can be a few GB of RAM)
    ## and is larger if names are larger. We are grabbing two lines at a time.
    alldereps = {}
    with open(derepfile, 'rb') as ioderep:
        dereps = itertools.izip(*[iter(ioderep)]*2)
        for namestr, seq in dereps:
            nnn, sss = [i.strip() for i in namestr, seq]
            alldereps[nnn[1:]] = sss

    ## store observed seeds (this could count up to >million in bad data sets)
    seedsseen = set()

    ## Iterate through the usort file grabbing matches to build clusters
    with open(usort, 'rb') as insort:
        ## iterator, seed null, seqlist null
        isort = iter(insort)
        lastseed = 0
        fseqs = []
        seqlist = []
        seqsize = 0
        while 1:
            ## grab the next line
            try:
                hit, seed, _, ind, ori, _ = isort.next().strip().split()
                LOGGER.debug(">{} {} {}".format(hit, seed, ori, seq))
            except StopIteration:
                break

            ## same seed, append match
            if seed != lastseed:
                seedsseen.add(seed)
                ## store the last cluster (fseq), count it, and clear fseq
                if fseqs:
                    ## sort fseqs by derep after pulling out the seed
                    fseqs = [fseqs[0]] + sorted(fseqs[1:], key=lambda x: \
                        int(x.split(";size=")[1].split(";")[0]), reverse=True)
                    seqlist.append("\n".join(fseqs))
                    seqsize += 1
                    fseqs = []

                ## occasionally write/dump stored clusters to file and clear mem
                if not seqsize % 10000:
                    if seqlist:
                        clustsout.write("\n//\n//\n".join(seqlist)+"\n//\n//\n")
                        ## reset list and counter
                        seqlist = []

                ## store the new seed on top of fseq list
                fseqs.append(">{}*\n{}".format(seed, alldereps[seed]))
                lastseed = seed

            ## add match to the seed
            ## revcomp if orientation is reversed (comp preserves nnnn)
            if ori == "-":
                seq = comp(alldereps[hit])[::-1]
            else:
                seq = alldereps[hit]
            ## only save if not too many indels
            if int(ind) <= maxindels:
                fseqs.append(">{}{}\n{}".format(hit, ori, seq))
            else:
                LOGGER.info("filtered by maxindels: %s %s", ind, seq)

    ## write whatever is left over to the clusts file
    if fseqs:
        seqlist.append("\n".join(fseqs))
    if seqlist:
        clustsout.write("\n//\n//\n".join(seqlist)+"\n//\n//\n")

    ## now write the seeds that had no hits. Make dict from htemp
    with open(hhandle, 'rb') as iotemp:
        nohits = itertools.izip(*[iter(iotemp)]*2)
        seqlist = []
        seqsize = 0
        while 1:
            try:
                nnn, _ = [i.strip() for i in nohits.next()]
            except StopIteration:
                break

            ## occasionally write to file
            if not seqsize % 10000:
                if seqlist:
                    clustsout.write("\n//\n//\n".join(seqlist)+"\n//\n//\n")
                    ## reset list and counter
                    seqlist = []

            ## append to list if new seed
            if nnn[1:] not in seedsseen:
                seqlist.append("{}*\n{}".format(nnn, alldereps[nnn[1:]]))
                seqsize += 1

    ## write whatever is left over to the clusts file
    if seqlist:
        clustsout.write("\n//\n//\n".join(seqlist))#+"\n//\n//\n")

    ## close the file handle
    clustsout.close()
    del alldereps


Instruction:
Can you implement a function in Python 3 that
sets up directories for step 3 data

Response:
def setup_dirs(data):
    """ sets up directories for step3 data """
    ## make output folder for clusters
    pdir = os.path.realpath(data.paramsdict["project_dir"])
    data.dirs.clusts = os.path.join(pdir, "{}_clust_{}"\
                       .format(data.name, data.paramsdict["clust_threshold"]))
    if not os.path.exists(data.dirs.clusts):
        os.mkdir(data.dirs.clusts)

    ## make a tmpdir for align files
    data.tmpdir = os.path.abspath(os.path.expanduser(
        os.path.join(pdir, data.name+'-tmpalign')))
    if not os.path.exists(data.tmpdir):
        os.mkdir(data.tmpdir)

    ## If ref mapping, init samples and make the refmapping output directory.
    if not data.paramsdict["assembly_method"] == "denovo":
        ## make output directory for read mapping process
        data.dirs.refmapping = os.path.join(pdir, "{}_refmapping".format(data.name))
        if not os.path.exists(data.dirs.refmapping):
            os.mkdir(data.dirs.refmapping)


Instruction:
How would you code a function in Python 3 to
create a DAG of pre - align jobs for each sample.

Response:
def new_apply_jobs(data, samples, ipyclient, nthreads, maxindels, force):
    """
    Create a DAG of prealign jobs to be run in order for each sample. Track
    Progress, report errors. Each assembly method has a slightly different
    DAG setup, calling different functions.
    """

    ## is datatype gbs? used in alignment-trimming by align_and_parse()
    is_gbs = bool("gbs" in data.paramsdict["datatype"])

    ## Two view objects, threaded and unthreaded
    lbview = ipyclient.load_balanced_view()
    start = time.time()
    elapsed = datetime.timedelta(seconds=int(time.time()-start))
    firstfunc = "derep_concat_split"
    printstr = " {}    | {} | s3 |".format(PRINTSTR[firstfunc], elapsed)
    #printstr = " {}      | {} | s3 |".format(PRINTSTR[], elapsed)
    progressbar(10, 0, printstr, spacer=data._spacer)

    ## TODO: for HPC systems this should be done to make sure targets are spread
    ## among different nodes.
    if nthreads:
        if nthreads < len(ipyclient.ids):
            thview = ipyclient.load_balanced_view(targets=ipyclient.ids[::nthreads])
        elif nthreads == 1:
            thview = ipyclient.load_balanced_view()
        else:
            if len(ipyclient) > 40:
                thview = ipyclient.load_balanced_view(targets=ipyclient.ids[::4])
            else:
                thview = ipyclient.load_balanced_view(targets=ipyclient.ids[::2])

    ## get list of jobs/dependencies as a DAG for all pre-align funcs.
    dag, joborder = build_dag(data, samples)

    ## dicts for storing submitted jobs and results
    results = {}

    ## submit jobs to the engines in single or threaded views. The topological
    ## sort makes sure jobs are input with all dependencies found.
    for node in nx.topological_sort(dag):
        ## get list of async results leading to this job
        deps = [results.get(n) for n in dag.predecessors(node)]
        deps = ipp.Dependency(dependencies=deps, failure=True)

        ## get func, sample, and args for this func (including [data, sample])
        funcstr, chunk, sname = node.split("-", 2)
        func = FUNCDICT[funcstr]
        sample = data.samples[sname]

        ## args vary depending on the function
        if funcstr in ["derep_concat_split", "cluster"]:
            args = [data, sample, nthreads, force]
        elif funcstr in ["mapreads"]:
            args = [data, sample, nthreads, force]
        elif funcstr in ["build_clusters"]:
            args = [data, sample, maxindels]
        elif funcstr in ["muscle_align"]:
            handle = os.path.join(data.tmpdir,
                        "{}_chunk_{}.ali".format(sample.name, chunk))
            args = [handle, maxindels, is_gbs]
        else:
            args = [data, sample]

        # submit and store AsyncResult object. Some jobs are threaded.
        if nthreads and (funcstr in THREADED_FUNCS):
            #LOGGER.info('submitting %s to %s-threaded view', funcstr, nthreads)
            with thview.temp_flags(after=deps, block=False):
                results[node] = thview.apply(func, *args)
        else:
            #LOGGER.info('submitting %s to single-threaded view', funcstr)
            with lbview.temp_flags(after=deps, block=False):
                results[node] = lbview.apply(func, *args)

    ## track jobs as they finish, abort if someone fails. This blocks here
    ## until all jobs are done. Keep track of which samples have failed so
    ## we only print the first error message.
    sfailed = set()
    for funcstr in joborder + ["muscle_align", "reconcat"]:
        errfunc, sfails, msgs = trackjobs(funcstr, results, spacer=data._spacer)
        LOGGER.info("{}-{}-{}".format(errfunc, sfails, msgs))
        if errfunc:
            for sidx in xrange(len(sfails)):
                sname = sfails[sidx]
                errmsg = msgs[sidx]
                if sname not in sfailed:
                    print("  sample [{}] failed. See error in ./ipyrad_log.txt"\
                          .format(sname))
                    LOGGER.error("sample [%s] failed in step [%s]; error: %s",
                                  sname, errfunc, errmsg)
                    sfailed.add(sname)

    ## Cleanup of successful samples, skip over failed samples
    badaligns = {}
    for sample in samples:
        ## The muscle_align step returns the number of excluded bad alignments
        for async in results:
            func, chunk, sname = async.split("-", 2)
            if (func == "muscle_align") and (sname == sample.name):
                if results[async].successful():
                    badaligns[sample] = int(results[async].get())

    ## for the samples that were successful:
    for sample in badaligns:
        ## store the result
        sample.stats_dfs.s3.filtered_bad_align = badaligns[sample]
        ## store all results
        try:
            sample_cleanup(data, sample)
        except Exception as inst:
            msg = "  Sample {} failed this step. See ipyrad_log.txt.\
                  ".format(sample.name)
            print(msg)
            LOGGER.error("%s - %s", sample.name, inst)

    ## store the results to data
    data_cleanup(data)


Instruction:
Create a Python 3 function for
building a directed acyclic graph describing jobs to be run in order.

Response:
def build_dag(data, samples):
    """
    build a directed acyclic graph describing jobs to be run in order.
    """

    ## Create DAGs for the assembly method being used, store jobs in nodes
    snames = [i.name for i in samples]
    dag = nx.DiGraph()

    ## get list of pre-align jobs from globals based on assembly method
    joborder = JOBORDER[data.paramsdict["assembly_method"]]

    ## WHICH JOBS TO RUN: iterate over the sample names
    for sname in snames:
        ## append pre-align job for each sample to nodes list
        for func in joborder:
            dag.add_node("{}-{}-{}".format(func, 0, sname))

        ## append align func jobs, each will have max 10
        for chunk in xrange(10):
            dag.add_node("{}-{}-{}".format("muscle_align", chunk, sname))

        ## append final reconcat jobs
        dag.add_node("{}-{}-{}".format("reconcat", 0, sname))

    ## ORDER OF JOBS: add edges/dependency between jobs: (first-this, then-that)
    for sname in snames:
        for sname2 in snames:
            ## enforce that clust/map cannot start until derep is done for ALL
            ## samples. This is b/c...
            dag.add_edge("{}-{}-{}".format(joborder[0], 0, sname2),
                         "{}-{}-{}".format(joborder[1], 0, sname))

        ## add remaining pre-align jobs
        for idx in xrange(2, len(joborder)):
            dag.add_edge("{}-{}-{}".format(joborder[idx-1], 0, sname),
                         "{}-{}-{}".format(joborder[idx], 0, sname))

        ## Add 10 align jobs, none of which can start until all chunker jobs
        ## are finished. Similarly, reconcat jobs cannot start until all align
        ## jobs are finished.
        for sname2 in snames:
            for chunk in range(10):
                dag.add_edge("{}-{}-{}".format("muscle_chunker", 0, sname2),
                             "{}-{}-{}".format("muscle_align", chunk, sname))
                ## add that the final reconcat job can't start until after
                ## each chunk of its own sample has finished aligning.
                dag.add_edge("{}-{}-{}".format("muscle_align", chunk, sname),
                             "{}-{}-{}".format("reconcat", 0, sname))
    ## return the dag
    return dag, joborder


Instruction:
Create a Python 3 function for
plotting the DAG with the given results.

Response:
def _plot_dag(dag, results, snames):
    """
    makes plot to help visualize the DAG setup. For developers only.
    """
    try:
        import matplotlib.pyplot as plt
        from matplotlib.dates import date2num
        from matplotlib.cm import gist_rainbow

        ## first figure is dag layout
        plt.figure("dag_layout", figsize=(10, 10))
        nx.draw(dag,
                pos=nx.spring_layout(dag),
                node_color='pink',
                with_labels=True)
        plt.savefig("./dag_layout.png", bbox_inches='tight', dpi=200)

        ## second figure is times for steps
        pos = {}
        colors = {}

        for node in dag:
            #jobkey = "{}-{}".format(node, sample)
            mtd = results[node].metadata
            start = date2num(mtd.started)
            #runtime = date2num(md.completed)# - start
            ## sample id to separate samples on x-axis
            _, _, sname = node.split("-", 2)
            sid = snames.index(sname)
            ## 1e6 to separate on y-axis
            pos[node] = (start+sid, start*1e6)
            colors[node] = mtd.engine_id

        ## x just spaces out samples;
        ## y is start time of each job with edge leading to next job
        ## color is the engine that ran the job
        ## all jobs were submitted as 3 second wait times
        plt.figure("dag_starttimes", figsize=(10, 16))
        nx.draw(dag, pos,
                node_list=colors.keys(),
                node_color=colors.values(),
                cmap=gist_rainbow,
                with_labels=True)
        plt.savefig("./dag_starttimes.png", bbox_inches='tight', dpi=200)

    except Exception as inst:
        LOGGER.warning(inst)


Instruction:
Write a Python 3 script to
track the jobs that have been submitted from a list of engine jobs. Returns whether any of the jobs failed or not.

Response:
def trackjobs(func, results, spacer):
    """
    Blocks and prints progress for just the func being requested from a list
    of submitted engine jobs. Returns whether any of the jobs failed.

    func = str
    results = dict of asyncs
    """

    ## TODO: try to insert a better way to break on KBD here.
    LOGGER.info("inside trackjobs of %s", func)

    ## get just the jobs from results that are relevant to this func
    asyncs = [(i, results[i]) for i in results if i.split("-", 2)[0] == func]

    ## progress bar
    start = time.time()
    while 1:
        ## how many of this func have finished so far
        ready = [i[1].ready() for i in asyncs]
        elapsed = datetime.timedelta(seconds=int(time.time()-start))
        printstr = " {}    | {} | s3 |".format(PRINTSTR[func], elapsed)
        progressbar(len(ready), sum(ready), printstr, spacer=spacer)
        time.sleep(0.1)
        if len(ready) == sum(ready):
            print("")
            break

    sfails = []
    errmsgs = []
    for job in asyncs:
        if not job[1].successful():
            sfails.append(job[0])
            errmsgs.append(job[1].result())

    return func, sfails, errmsgs


Instruction:
Here you have a function in Python 3, explain what it does
def data_cleanup(data):
    data.stats_dfs.s3 = data._build_stat("s3")
    data.stats_files.s3 = os.path.join(data.dirs.clusts, "s3_cluster_stats.txt")
    with io.open(data.stats_files.s3, 'w') as outfile:
        data.stats_dfs.s3.to_string(
            buf=outfile,
            formatters={
                'merged_pairs':'{:.0f}'.format,
                'clusters_total':'{:.0f}'.format,
                'clusters_hidepth':'{:.0f}'.format,
                'filtered_bad_align':'{:.0f}'.format,
                'avg_depth_stat':'{:.2f}'.format,
                'avg_depth_mj':'{:.2f}'.format,
                'avg_depth_total':'{:.2f}'.format,
                'sd_depth_stat':'{:.2f}'.format,
                'sd_depth_mj':'{:.2f}'.format,
                'sd_depth_total':'{:.2f}'.format
            })

Response: cleanup statswriting function for Assembly obj


Instruction:
Can you write a function in Python 3 where it
concatenates multiple sample. files.

Response:
def concat_multiple_edits(data, sample):
    """
    if multiple fastq files were appended into the list of fastqs for samples
    then we merge them here before proceeding.
    """

    ## if more than one tuple in fastq list
    if len(sample.files.edits) > 1:
        ## create a cat command to append them all (doesn't matter if they
        ## are gzipped, cat still works). Grab index 0 of tuples for R1s.
        cmd1 = ["cat"] + [i[0] for i in sample.files.edits]

        ## write to new concat handle
        conc1 = os.path.join(data.dirs.edits, sample.name+"_R1_concatedit.fq.gz")
        with open(conc1, 'w') as cout1:
            proc1 = sps.Popen(cmd1, stderr=sps.STDOUT, stdout=cout1, close_fds=True)
            res1 = proc1.communicate()[0]
        if proc1.returncode:
            raise IPyradWarningExit("error in: %s, %s", cmd1, res1)

        ## Only set conc2 if R2 actually exists
        conc2 = 0
        if os.path.exists(str(sample.files.edits[0][1])):
            cmd2 = ["cat"] + [i[1] for i in sample.files.edits]
            conc2 = os.path.join(data.dirs.edits, sample.name+"_R2_concatedit.fq.gz")
            with gzip.open(conc2, 'w') as cout2:
                proc2 = sps.Popen(cmd2, stderr=sps.STDOUT, stdout=cout2, close_fds=True)
                res2 = proc2.communicate()[0]
            if proc2.returncode:
                raise IPyradWarningExit("error in: %s, %s", cmd2, res2)

        ## store new file handles
        sample.files.edits = [(conc1, conc2)]
    return sample.files.edits


Instruction:
Explain what the following Python 3 code does
def cluster(data, sample, nthreads, force):

    ## get the dereplicated reads
    if "reference" in data.paramsdict["assembly_method"]:
        derephandle = os.path.join(data.dirs.edits, sample.name+"-refmap_derep.fastq")
        ## In the event all reads for all samples map successfully then clustering
        ## the unmapped reads makes no sense, so just bail out.
        if not os.stat(derephandle).st_size:
            ## In this case you do have to create empty, dummy vsearch output
            ## files so building_clusters will not fail.
            uhandle = os.path.join(data.dirs.clusts, sample.name+".utemp")
            usort = os.path.join(data.dirs.clusts, sample.name+".utemp.sort")
            hhandle = os.path.join(data.dirs.clusts, sample.name+".htemp")
            for f in [uhandle, usort, hhandle]:
                open(f, 'a').close()
            return
    else:
        derephandle = os.path.join(data.dirs.edits, sample.name+"_derep.fastq")

    ## create handles for the outfiles
    uhandle = os.path.join(data.dirs.clusts, sample.name+".utemp")
    temphandle = os.path.join(data.dirs.clusts, sample.name+".htemp")

    ## If derep file doesn't exist then bail out
    if not os.path.isfile(derephandle):
        LOGGER.warn("Bad derephandle - {}".format(derephandle))
        raise IPyradError("Input file for clustering doesn't exist - {}"\
                        .format(derephandle))

    ## testing one sample fail
    #if sample.name == "1C_0":
    #    x

    ## datatype specific optimization
    ## minsl: the percentage of the seed that must be matched
    ##    smaller values for RAD/ddRAD where we might want to combine, say 50bp
    ##    reads and 100bp reads in the same analysis.
    ## query_cov: the percentage of the query sequence that must match seed
    ##    smaller values are needed for gbs where only the tips might overlap
    ##    larger values for pairgbs where they should overlap near completely
    ##    small minsl and high query cov allows trimmed reads to match to untrim
    ##    seed for rad/ddrad/pairddrad.
    strand = "plus"
    cov = 0.75
    minsl = 0.5
    if data.paramsdict["datatype"] in ["gbs", "2brad"]:
        strand = "both"
        cov = 0.5
        minsl = 0.5
    elif data.paramsdict["datatype"] == 'pairgbs':
        strand = "both"
        cov = 0.75
        minsl = 0.75

    ## If this value is not null (which is the default) then override query cov
    if data._hackersonly["query_cov"]:
        cov = str(data._hackersonly["query_cov"])
        assert float(cov) <= 1, "query_cov must be <= 1.0"

    ## get call string
    cmd = [ipyrad.bins.vsearch,
           "-cluster_smallmem", derephandle,
           "-strand", strand,
           "-query_cov", str(cov),
           "-id", str(data.paramsdict["clust_threshold"]),
           "-minsl", str(minsl),
           "-userout", uhandle,
           "-userfields", "query+target+id+gaps+qstrand+qcov",
           "-maxaccepts", "1",
           "-maxrejects", "0",
           "-threads", str(nthreads),
           "-notmatched", temphandle,
           "-fasta_width", "0",
           "-fastq_qmax", "100",
           "-fulldp",
           "-usersort"]

    ## not sure what the benefit of this option is exactly, needs testing,
    ## might improve indel detection on left side, but we don't want to enforce
    ## aligning on left side if not necessarily, since quality trimmed reads
    ## might lose bases on left side in step2 and no longer align.
    #if data.paramsdict["datatype"] in ["rad", "ddrad", "pairddrad"]:
    #    cmd += ["-leftjust"]

    ## run vsearch
    LOGGER.debug("%s", cmd)
    proc = sps.Popen(cmd, stderr=sps.STDOUT, stdout=sps.PIPE, close_fds=True)

    ## This is long running so we wrap it to make sure we can kill it
    try:
        res = proc.communicate()[0]
    except KeyboardInterrupt:
        proc.kill()
        raise KeyboardInterrupt

    ## check for errors
    if proc.returncode:
        LOGGER.error("error %s: %s", cmd, res)
        raise IPyradWarningExit("cmd {}: {}".format(cmd, res))

Response: Cluster the reads in the input file.


Instruction:
Can you generate the documentation for the following Python 3 function
def muscle_chunker(data, sample):
    ## log our location for debugging
    LOGGER.info("inside muscle_chunker")

    ## only chunk up denovo data, refdata has its own chunking method which
    ## makes equal size chunks, instead of uneven chunks like in denovo
    if data.paramsdict["assembly_method"] != "reference":
        ## get the number of clusters
        clustfile = os.path.join(data.dirs.clusts, sample.name+".clust.gz")
        with iter(gzip.open(clustfile, 'rb')) as clustio:
            nloci = sum(1 for i in clustio if "//" in i) // 2
            #tclust = clustio.read().count("//")//2
            optim = (nloci//20) + (nloci%20)
            LOGGER.info("optim for align chunks: %s", optim)

        ## write optim clusters to each tmp file
        clustio = gzip.open(clustfile, 'rb')
        inclusts = iter(clustio.read().strip().split("//\n//\n"))

        ## splitting loci so first file is smaller and last file is bigger
        inc = optim // 10
        for idx in range(10):
            ## how big is this chunk?
            this = optim + (idx * inc)
            left = nloci-this
            if idx == 9:
                ## grab everything left
                grabchunk = list(itertools.islice(inclusts, int(1e9)))
            else:
                ## grab next chunks-worth of data
                grabchunk = list(itertools.islice(inclusts, this))
                nloci = left

            ## write the chunk to file
            tmpfile = os.path.join(data.tmpdir, sample.name+"_chunk_{}.ali".format(idx))
            with open(tmpfile, 'wb') as out:
                out.write("//\n//\n".join(grabchunk))

        ## write the chunk to file
        #grabchunk = list(itertools.islice(inclusts, left))
        #if grabchunk:
        #    tmpfile = os.path.join(data.tmpdir, sample.name+"_chunk_9.ali")
        #    with open(tmpfile, 'a') as out:
        #        out.write("\n//\n//\n".join(grabchunk))
        clustio.close()

Response: This function splits the muscle alignment into chunks of size 10 and 10.


Instruction:
How would you implement a function in Python 3 that
reconciles the aligned reads and writes the new cluster files to the cluster files

Response:
def reconcat(data, sample):
    """ takes aligned chunks (usually 10) and concatenates them """

    try:
        ## get chunks
        chunks = glob.glob(os.path.join(data.tmpdir,
                 sample.name+"_chunk_[0-9].aligned"))

        ## sort by chunk number, cuts off last 8 =(aligned)
        chunks.sort(key=lambda x: int(x.rsplit("_", 1)[-1][:-8]))
        LOGGER.info("chunk %s", chunks)
        ## concatenate finished reads
        sample.files.clusters = os.path.join(data.dirs.clusts,
                                             sample.name+".clustS.gz")
        ## reconcats aligned clusters
        with gzip.open(sample.files.clusters, 'wb') as out:
            for fname in chunks:
                with open(fname) as infile:
                    dat = infile.read()
                    ## avoids mess if last chunk was empty
                    if dat.endswith("\n"):
                        out.write(dat+"//\n//\n")
                    else:
                        out.write(dat+"\n//\n//\n")
                os.remove(fname)
    except Exception as inst:
        LOGGER.error("Error in reconcat {}".format(inst))
        raise


Instruction:
Implement a function in Python 3 to
concatenate the reads in the Assembly into a single Assembly.

Response:
def derep_concat_split(data, sample, nthreads, force):
    """
    Running on remote Engine. Refmaps, then merges, then dereplicates,
    then denovo clusters reads.
    """

    ## report location for debugging
    LOGGER.info("INSIDE derep %s", sample.name)

    ## MERGED ASSEMBIES ONLY:
    ## concatenate edits files within Samples. Returns a new sample.files.edits
    ## with the concat file. No change if not merged Assembly.
    mergefile = os.path.join(data.dirs.edits, sample.name+"_merged_.fastq")
    if not force:
        if not os.path.exists(mergefile):
            sample.files.edits = concat_multiple_edits(data, sample)
        else:
            LOGGER.info("skipped concat_multiple_edits: {} exists"\
                        .format(mergefile))
    else:
        sample.files.edits = concat_multiple_edits(data, sample)

    ## PAIRED DATA ONLY:
    ## Denovo: merge or concat fastq pairs [sample.files.pairs]
    ## Reference: only concat fastq pairs  []
    ## Denovo + Reference: ...
    if 'pair' in data.paramsdict['datatype']:
        ## the output file handle for merged reads

        ## modify behavior of merging vs concating if reference
        if "reference" in data.paramsdict["assembly_method"]:
            nmerged = merge_pairs(data, sample.files.edits, mergefile, 0, 0)
        else:
            nmerged = merge_pairs(data, sample.files.edits, mergefile, 1, 1)

        ## store results
        sample.files.edits = [(mergefile, )]
        sample.stats.reads_merged = nmerged

    ## 3rad uses random adapters to identify pcr duplicates. We will
    ## remove pcr dupes here. Basically append the radom adapter to
    ## each sequence, do a regular old vsearch derep, then trim
    ## off the adapter, and push it down the pipeline. This will
    ## remove all identical seqs with identical random i5 adapters.
    if "3rad" in data.paramsdict["datatype"]:
        declone_3rad(data, sample)
        derep_and_sort(data,
                os.path.join(data.dirs.edits, sample.name+"_declone.fastq"),
                os.path.join(data.dirs.edits, sample.name+"_derep.fastq"),
                nthreads)
    else:
        ## convert fastq to fasta, then derep and sort reads by their size.
        ## we pass in only one file b/c paired should be merged by now.
        derep_and_sort(data,
                sample.files.edits[0][0],
                os.path.join(data.dirs.edits, sample.name+"_derep.fastq"),
                nthreads)


Instruction:
Create a Python 3 function to
run the major functions for clustering within samples

Response:
def run(data, samples, noreverse, maxindels, force, ipyclient):
    """ run the major functions for clustering within samples """

    ## list of samples to submit to queue
    subsamples = []

    ## if sample is already done skip
    for sample in samples:
        ## If sample not in state 2 don't try to cluster it.
        if sample.stats.state < 2:
            print("""\
    Sample not ready for clustering. First run step2 on sample: {}""".\
    format(sample.name))
            continue

        if not force:
            if sample.stats.state >= 3:
                print("""\
    Skipping {}; aleady clustered. Use force to re-cluster""".\
    format(sample.name))
            else:
                if sample.stats.reads_passed_filter:
                    subsamples.append(sample)
        else:
            ## force to overwrite
            if sample.stats.reads_passed_filter:
                subsamples.append(sample)

    ## run subsamples
    if not subsamples:
        print("  No Samples ready to be clustered. First run step2().")

    else:
        ## arguments to apply_jobs, inst catches exceptions
        try:
            ## make dirs that are needed including tmpdir
            setup_dirs(data)

            ## if refmapping make filehandles that will be persistent
            if not data.paramsdict["assembly_method"] == "denovo":
                for sample in subsamples:
                    refmap_init(data, sample, force)

                    ## set thread-count to 2 for paired-data
                    nthreads = 2
            ## set thread-count to 1 for single-end data
            else:
                nthreads = 1

            ## overwrite nthreads if value in _ipcluster dict
            if "threads" in data._ipcluster.keys():
                nthreads = int(data._ipcluster["threads"])

                ## if more CPUs than there are samples then increase threads
                _ncpus = len(ipyclient)
                if _ncpus > 2*len(data.samples):
                    nthreads *= 2

            ## submit jobs to be run on cluster
            args = [data, subsamples, ipyclient, nthreads, maxindels, force]
            new_apply_jobs(*args)

        finally:
            ## this can fail if jobs were not stopped properly and are still
            ## writing to tmpdir. don't cleanup if debug is on.
            try:
                log_level = logging.getLevelName(LOGGER.getEffectiveLevel())
                if not log_level == "DEBUG":

                    if os.path.exists(data.tmpdir):
                        shutil.rmtree(data.tmpdir)
                    ## get all refmap_derep.fastqs
                    rdereps = glob.glob(os.path.join(data.dirs.edits, "*-refmap_derep.fastq"))
                    ## Remove the unmapped fastq files
                    for rmfile in rdereps:
                        os.remove(rmfile)

            except Exception as _:
                LOGGER.warning("failed to cleanup files/dirs")


Instruction:
How would you implement a function in Python 3 that
parses the params. txt file args create and return Assembly object.

Response:
def parse_params(args):
    """ Parse the params file args, create and return Assembly object."""

    ## check that params.txt file is correctly formatted.
    try:
        with open(args.params) as paramsin:
            plines = paramsin.readlines()
    except IOError as _:
        sys.exit("  No params file found")

    ## check header: big version changes can be distinguished by the header
    legacy_version = 0
    try:
        ## try to update the Assembly ...
        legacy_version = 1
        if not len(plines[0].split()[0]) == 7:
            raise IPyradWarningExit("""
        Error: file '{}' is not compatible with ipyrad v.{}.
        Please create and update a new params file using the -n argument.
        For info on which parameters have changed see the changelog:
        (http://ipyrad.readthedocs.io/releasenotes.html)
        """.format(args.params, ip.__version__))

    except IndexError:
        raise IPyradWarningExit("""
        Error: Params file should not have any empty lines at the top
        of the file. Verify there are no blank lines and rerun ipyrad.
        Offending file - {}
        """.format(args.params))

    ## update and backup
    if legacy_version:
        #which version...
        #update_to_6()
        pass

    ## make into a dict. Ignore blank lines at the end of file
    ## Really this will ignore all blank lines
    items = [i.split("##")[0].strip() for i in plines[1:] if not i.strip() == ""]

    #keys = [i.split("]")[-2][-1] for i in plines[1:]]
    #keys = range(len(plines)-1)
    keys = ip.Assembly('null', quiet=True).paramsdict.keys()
    parsedict = {str(i):j for i, j in zip(keys, items)}
    return parsedict


Instruction:
Write a Python 3 script to
load the Assembly and print the stats to screen.

Response:
def showstats(parsedict):
    """ loads assembly or dies, and print stats to screen """

    #project_dir = parsedict['1']
    project_dir = parsedict["project_dir"]
    if not project_dir:
        project_dir = "./"
    ## Be nice if somebody also puts in the file extension
    #assembly_name = parsedict['0']
    assembly_name = parsedict["assembly_name"]
    my_assembly = os.path.join(project_dir, assembly_name)

    ## If the project_dir doesn't exist don't even bother trying harder.
    if not os.path.isdir(project_dir):
        msg = """
    Trying to print stats for Assembly ({}) that doesn't exist. You must
    first run steps before you can show results.
    """.format(project_dir)
        sys.exit(msg)

    if not assembly_name:
        msg = """
    Assembly name is not set in params.txt, meaning it was either changed or
    erased since the Assembly was started. Please restore the original name.
    You can find the name of your Assembly in the "project dir": {}.
    """.format(project_dir)
        raise IPyradError(msg)

    data = ip.load_json(my_assembly, quiet=True, cli=True)

    print("\nSummary stats of Assembly {}".format(data.name) \
         +"\n------------------------------------------------")

    if not data.stats.empty:
        print(data.stats)
        print("\n\nFull stats files"\
         +"\n------------------------------------------------")

        fullcurdir = os.path.realpath(os.path.curdir)
        for i in range(1, 8):
            #enumerate(sorted(data.stats_files)):
            key = "s"+str(i)
            try:
                val = data.stats_files[key]
                val = val.replace(fullcurdir, ".")
                print("step {}: {}".format(i, val))
            except (KeyError, AttributeError):
                print("step {}: None".format(i))
        print("\n")
    else:
        print("No stats to display")


Instruction:
Can you create a Python 3 function that
loads the passed in assembly and create a branch.

Response:
def branch_assembly(args, parsedict):
    """
    Load the passed in assembly and create a branch. Copy it
    to a new assembly, and also write out the appropriate params.txt
    """

    ## Get the current assembly
    data = getassembly(args, parsedict)

    ## get arguments to branch command
    bargs = args.branch

    ## get new name, trim off .txt if it was accidentally added
    newname = bargs[0]
    if newname.endswith(".txt"):
        newname = newname[:-4]

    ## look for subsamples
    if len(bargs) > 1:
        ## Branching and subsampling at step 6 is a bad idea, it messes up
        ## indexing into the hdf5 cluster file. Warn against this.
        if any([x.stats.state == 6 for x in data.samples.values()]):
            pass
            ## TODODODODODO
            #print("wat")

        ## are we removing or keeping listed samples?
        subsamples = bargs[1:]

        ## drop the matching samples
        if bargs[1] == "-":
            ## check drop names
            fails = [i for i in subsamples[1:] if i not in data.samples.keys()]
            if any(fails):
                raise IPyradWarningExit("\
                    \n  Failed: unrecognized names requested, check spelling:\n  {}"\
                    .format("\n  ".join([i for i in fails])))
            print("  dropping {} samples".format(len(subsamples)-1))
            subsamples = list(set(data.samples.keys()) - set(subsamples))

        ## If the arg after the new param name is a file that exists
        if os.path.exists(bargs[1]):
            new_data = data.branch(newname, infile=bargs[1])
        else:
            new_data = data.branch(newname, subsamples)

    ## keeping all samples
    else:
        new_data = data.branch(newname, None)

    print("  creating a new branch called '{}' with {} Samples".\
             format(new_data.name, len(new_data.samples)))

    print("  writing new params file to {}"\
            .format("params-"+new_data.name+".txt\n"))
    new_data.write_params("params-"+new_data.name+".txt", force=args.force)


Instruction:
Write a Python 3 function for
merging all given assemblies into a new assembly. Copies the params from the first passed in extant assembly. this function is called with the ipyrad -m flag. You must pass it at least 3 values, the first is a new assembly name (a new `param-newname.txt` will be created). The second and third args must be params files for currently existing assemblies. Any args beyond the third must also be params file for extant assemblies.

Response:
def merge_assemblies(args):
    """
    merge all given assemblies into a new assembly. Copies the params
    from the first passed in extant assembly. this function is called
    with the ipyrad -m flag. You must pass it at least 3 values, the first
    is a new assembly name (a new `param-newname.txt` will be created).
    The second and third args must be params files for currently existing
    assemblies. Any args beyond the third must also be params file for
    extant assemblies.
    """
    print("\n  Merging assemblies: {}".format(args.merge[1:]))

    ## Make sure there are the right number of args
    if len(args.merge) < 3:
        sys.exit(_WRONG_NUM_CLI_MERGE)

    ## Make sure the first arg isn't a params file, i could see someone doing it
    newname = args.merge[0]
    if os.path.exists(newname) and "params-" in newname:
        sys.exit(_WRONG_ORDER_CLI_MERGE)

    ## Make sure first arg will create a param file that doesn't already exist
    if os.path.exists("params-" + newname + ".txt") and not args.force:
        sys.exit(_NAME_EXISTS_MERGE.format("params-" + newname + ".txt"))

    ## Make sure the rest of the args are params files that already exist
    assemblies_to_merge = args.merge[1:]
    for assembly in assemblies_to_merge:
        if not os.path.exists(assembly):
            sys.exit(_DOES_NOT_EXIST_MERGE.format(assembly))

    ## Get assemblies for each of the passed in params files.
    ## We're recycling some of the machinery for loading assemblies here
    assemblies = []
    for params_file in args.merge[1:]:
        args.params = params_file
        parsedict = parse_params(args)
        assemblies.append(getassembly(args, parsedict))

    ## Do the merge
    merged_assembly = ip.merge(newname, assemblies)

    ## Write out the merged assembly params file and report success
    merged_assembly.write_params("params-{}.txt".format(newname), force=args.force)

    print("\n  Merging succeeded. New params file for merged assembly:")
    print("\n    params-{}.txt\n".format(newname))


Instruction:
Given the following Python 3 function, write the documentation
def getassembly(args, parsedict):

    ## Creating an assembly with a full path in the name will "work"
    ## but it is potentially dangerous, so here we have assembly_name
    ## and assembly_file, name is used for creating new in cwd, file is
    ## used for loading existing.
    ##
    ## Be nice if the user includes the extension.
    #project_dir = ip.core.assembly._expander(parsedict['1'])
    #assembly_name = parsedict['0']
    project_dir = ip.core.assembly._expander(parsedict['project_dir'])
    assembly_name = parsedict['assembly_name']
    assembly_file = os.path.join(project_dir, assembly_name)

    ## Assembly creation will handle error checking  on
    ## the format of the assembly_name

    ## make sure the working directory exists.
    if not os.path.exists(project_dir):
        os.mkdir(project_dir)

    try:
        ## If 1 and force then go ahead and create a new assembly
        if ('1' in args.steps) and args.force:
            data = ip.Assembly(assembly_name, cli=True)
        else:
            data = ip.load_json(assembly_file, cli=True)
            data._cli = True

    except IPyradWarningExit as _:
        ## if no assembly is found then go ahead and make one
        if '1' not in args.steps:
            raise IPyradWarningExit(\
                "  Error: You must first run step 1 on the assembly: {}"\
                .format(assembly_file))
        else:
            ## create a new assembly object
            data = ip.Assembly(assembly_name, cli=True)

    ## for entering some params...
    for param in parsedict:

        ## trap assignment of assembly_name since it is immutable.
        if param == "assembly_name":
            ## Raise error if user tried to change assembly name
            if parsedict[param] != data.name:
                data.set_params(param, parsedict[param])
        else:
            ## all other params should be handled by set_params
            try:
                data.set_params(param, parsedict[param])
            except IndexError as _:
                print("  Malformed params file: {}".format(args.params))
                print("  Bad parameter {} - {}".format(param, parsedict[param]))
                sys.exit(-1)
    return data

Response: Loads an assembly or creates a new one and set its params from parsedict. Does not launch ipcluster.


Instruction:
Create a Python 3 function for
testing if there s a newer version and nag the user to upgrade.

Response:
def _check_version():
    """ Test if there's a newer version and nag the user to upgrade."""
    import urllib2
    from distutils.version import LooseVersion

    header = \
    "\n -------------------------------------------------------------"+\
    "\n  ipyrad [v.{}]".format(ip.__version__)+\
    "\n  Interactive assembly and analysis of RAD-seq data"+\
    "\n -------------------------------------------------------------"

    try:
        htmldat = urllib2.urlopen("https://anaconda.org/ipyrad/ipyrad").readlines()
        curversion = next((x for x in htmldat if "subheader" in x), None).split(">")[1].split("<")[0]
        if LooseVersion(ip.__version__) < LooseVersion(curversion):
            msg = """
  A new version of ipyrad is available (v.{}). To upgrade run:

    conda install -c ipyrad ipyrad\n""".format(curversion)
            print(header + "\n" + msg)
        else:
            pass
            #print("You are up to date")
    except Exception as inst:
        ## Always fail silently
        pass


Instruction:
Write a Python 3 function that can
calculate probability of binom

Response:
def get_binom(base1, base2, estE, estH):
    """
    return probability of base call
    """

    prior_homo = (1. - estH) / 2.
    prior_hete = estH

    ## calculate probs
    bsum = base1 + base2
    hetprob = scipy.misc.comb(bsum, base1)/(2. **(bsum))
    homoa = scipy.stats.binom.pmf(base2, bsum, estE)
    homob = scipy.stats.binom.pmf(base1, bsum, estE)

    ## calculate probs
    hetprob *= prior_hete
    homoa *= prior_homo
    homob *= prior_homo

    ## final
    probabilities = [homoa, homob, hetprob]
    bestprob = max(probabilities)/float(sum(probabilities))

    ## return
    if hetprob > homoa:
        return True, bestprob
    else:
        return False, bestprob


Instruction:
Write a Python 3 script to
remove all non - overlapping consensus seqs from the arrayed list.

Response:
def removerepeats(consens, arrayed):
    """
    Checks for interior Ns in consensus seqs and removes those that are at
    low depth, here defined as less than 1/3 of the average depth. The prop 1/3
    is chosen so that mindepth=6 requires 2 base calls that are not in [N,-].
    """

    ## default trim no edges
    consens = "".join(consens).replace("-", "N")

    ## split for pairs
    try:
        cons1, cons2 = consens.split("nnnn")
        split = consens.index("nnnn")
        arr1 = arrayed[:, :split]
        arr2 = arrayed[:, split+4:]
    except ValueError:
        cons1 = consens
        cons2 = ""
        arr1 = arrayed

    ## trim from left and right of cons1
    edges = [None, None]
    lcons = len(cons1)
    cons1 = cons1.lstrip("N")
    edges[0] = lcons - len(cons1)

    ## trim from right if nonzero
    lcons = len(cons1)
    cons1 = cons1.rstrip("N")
    if lcons - len(cons1):
        edges[1] = -1*(lcons - len(cons1))

    ## trim same from arrayed
    arr1 = arr1[:, edges[0]:edges[1]]

    ## trim from left and right of cons2 if present
    if cons2:
        ## trim from left and right of cons1
        edges = [None, None]
        lcons = len(cons2)
        cons2 = cons2.lstrip("N")
        edges[0] = lcons - len(cons2)

        ## trim from right if nonzero
        lcons = len(cons2)
        cons2 = cons2.rstrip("N")
        if lcons - len(cons2):
            edges[1] = -1*(lcons - len(cons2))

        ## trim same from arrayed
        arr2 = arr2[:, edges[0]:edges[1]]

        ## reconstitute pairs
        consens = cons1 + "nnnn" + cons2
        consens = np.array(list(consens))
        sep = np.array(arr1.shape[0]*[list("nnnn")])
        arrayed = np.hstack([arr1, sep, arr2])

    ## if single-end...
    else:
        consens = np.array(list(cons1))
        arrayed = arr1

    ## get column counts of Ns and -s
    ndepths = np.sum(arrayed == 'N', axis=0)
    idepths = np.sum(arrayed == '-', axis=0)

    ## get proportion of bases that are N- at each site
    nons = ((ndepths + idepths) / float(arrayed.shape[0])) >= 0.75
    ## boolean of whether base was called N
    isn = consens == "N"
    ## make ridx
    ridx = nons * isn

    ## apply filter
    consens = consens[~ridx]
    arrayed = arrayed[:, ~ridx]

    return consens, arrayed


Instruction:
Here you have a function in Python 3, explain what it does
def newconsensus(data, sample, tmpchunk, optim):
    ## do reference map funcs?
    isref = "reference" in data.paramsdict["assembly_method"]

    ## temporarily store the mean estimates to Assembly
    data._este = data.stats.error_est.mean()
    data._esth = data.stats.hetero_est.mean()

    ## get number relative to tmp file
    tmpnum = int(tmpchunk.split(".")[-1])

    ## prepare data for reading
    clusters = open(tmpchunk, 'rb')
    pairdealer = itertools.izip(*[iter(clusters)]*2)
    maxlen = data._hackersonly["max_fragment_length"]

    ## write to tmp cons to file to be combined later
    consenshandle = os.path.join(
        data.dirs.consens, sample.name+"_tmpcons."+str(tmpnum))
    tmp5 = consenshandle.replace("_tmpcons.", "_tmpcats.")
    with h5py.File(tmp5, 'w') as io5:
        io5.create_dataset("cats", (optim, maxlen, 4), dtype=np.uint32)
        io5.create_dataset("alls", (optim, ), dtype=np.uint8)
        io5.create_dataset("chroms", (optim, 3), dtype=np.int64)

        ## local copies to use to fill the arrays
        catarr = io5["cats"][:]
        nallel = io5["alls"][:]
        refarr = io5["chroms"][:]

    ## if reference-mapped then parse the fai to get index number of chroms
    if isref:
        fai = pd.read_csv(data.paramsdict["reference_sequence"] + ".fai",
                names=['scaffold', 'size', 'sumsize', 'a', 'b'],
                sep="\t")
        faidict = {j:i for i,j in enumerate(fai.scaffold)}

    ## store data for stats counters
    counters = {"name" : tmpnum,
                "heteros": 0,
                "nsites" : 0,
                "nconsens" : 0}

    ## store data for what got filtered
    filters = {"depth" : 0,
               "maxh" : 0,
               "maxn" : 0}

    ## store data for writing
    storeseq = {}

    ## set max limits
    if 'pair' in data.paramsdict["datatype"]:
        maxhet = sum(data.paramsdict["max_Hs_consens"])
        maxn = sum(data.paramsdict["max_Ns_consens"])
    else:
        maxhet = data.paramsdict["max_Hs_consens"][0]
        maxn = data.paramsdict["max_Ns_consens"][0]

    ## load the refmap dictionary if refmapping
    done = 0
    while not done:
        try:
            done, chunk = clustdealer(pairdealer, 1)
        except IndexError:
            raise IPyradError("clustfile formatting error in %s", chunk)

        if chunk:
            ## get names and seqs
            piece = chunk[0].strip().split("\n")
            names = piece[0::2]
            seqs = piece[1::2]

            ## pull replicate read info from seqs
            reps = [int(sname.split(";")[-2][5:]) for sname in names]

            ## IF this is a reference mapped read store the chrom and pos info
            ## -1 defaults to indicating an anonymous locus, since we are using
            ## the faidict as 0 indexed. If chrompos fails it defaults to -1
            ref_position = (-1, 0, 0)
            if isref:
                try:
                    ## parse position from name string
                    name, _, _ = names[0].rsplit(";", 2)
                    chrom, pos0, pos1 = name.rsplit(":", 2)

                    ## pull idx from .fai reference dict
                    chromint = faidict[chrom] + 1
                    ref_position = (int(chromint), int(pos0), int(pos1))

                except Exception as inst:
                    LOGGER.debug("Reference sequence chrom/pos failed for {}".format(names[0]))
                    LOGGER.debug(inst)

            ## apply read depth filter
            if nfilter1(data, reps):

                ## get stacks of base counts
                sseqs = [list(seq) for seq in seqs]
                arrayed = np.concatenate(
                    [[seq]*rep for seq, rep in zip(sseqs, reps)])
                arrayed = arrayed[:, :maxlen]

                ## get consens call for each site, applies paralog-x-site filter
                #consens = np.apply_along_axis(basecall, 0, arrayed, data)
                consens = basecaller(
                    arrayed,
                    data.paramsdict["mindepth_majrule"],
                    data.paramsdict["mindepth_statistical"],
                    data._esth,
                    data._este,
                    )

                ## apply a filter to remove low coverage sites/Ns that
                ## are likely sequence repeat errors. This is only applied to
                ## clusters that already passed the read-depth filter (1)
                if "N" in consens:
                    try:
                        consens, arrayed = removerepeats(consens, arrayed)

                    except ValueError as _:
                        LOGGER.info("Caught a bad chunk w/ all Ns. Skip it.")
                        continue

                ## get hetero sites
                hidx = [i for (i, j) in enumerate(consens) \
                            if j in list("RKSYWM")]
                nheteros = len(hidx)

                ## filter for max number of hetero sites
                if nfilter2(nheteros, maxhet):
                    ## filter for maxN, & minlen
                    if nfilter3(consens, maxn):
                        ## counter right now
                        current = counters["nconsens"]
                        ## get N alleles and get lower case in consens
                        consens, nhaps = nfilter4(consens, hidx, arrayed)
                        ## store the number of alleles observed
                        nallel[current] = nhaps

                        ## store a reduced array with only CATG
                        catg = np.array(\
                            [np.sum(arrayed == i, axis=0)  \
                            for i in list("CATG")],
                            dtype='uint32').T
                        catarr[current, :catg.shape[0], :] = catg
                        refarr[current] = ref_position

                        ## store the seqdata for tmpchunk
                        storeseq[counters["name"]] = "".join(list(consens))
                        counters["name"] += 1
                        counters["nconsens"] += 1
                        counters["heteros"] += nheteros
                    else:
                        #LOGGER.debug("@haplo")
                        filters['maxn'] += 1
                else:
                    #LOGGER.debug("@hetero")
                    filters['maxh'] += 1
            else:
                #LOGGER.debug("@depth")
                filters['depth'] += 1

    ## close infile io
    clusters.close()

    ## write final consens string chunk
    if storeseq:
        with open(consenshandle, 'wb') as outfile:
            outfile.write("\n".join([">"+sample.name+"_"+str(key)+"\n"+\
                                   str(storeseq[key]) for key in storeseq]))

    ## write to h5 array, this can be a bit slow on big data sets and is not
    ## currently convered by progressbar movement.
    with h5py.File(tmp5, 'a') as io5:
        io5["cats"][:] = catarr
        io5["alls"][:] = nallel
        io5["chroms"][:] = refarr
    del catarr
    del nallel
    del refarr

    ## return stats
    counters['nsites'] = sum([len(i) for i in storeseq.itervalues()])
    return counters, filters

Response: new faster replacement to consensus


Instruction:
How would you code a function in Python 3 to
call all sites in a locus array.

Response:
def basecaller(arrayed, mindepth_majrule, mindepth_statistical, estH, estE):
    """
    call all sites in a locus array.
    """

    ## an array to fill with consensus site calls
    cons = np.zeros(arrayed.shape[1], dtype=np.uint8)
    cons.fill(78)
    arr = arrayed.view(np.uint8)

    ## iterate over columns
    for col in xrange(arr.shape[1]):
        ## the site of focus
        carr = arr[:, col]

        ## make mask of N and - sites
        mask = carr == 45
        mask += carr == 78
        marr = carr[~mask]

        ## skip if only empties (e.g., N-)
        if not marr.shape[0]:
            cons[col] = 78

        ## skip if not variable
        elif np.all(marr == marr[0]):
            cons[col] = marr[0]

        ## estimate variable site call
        else:
            ## get allele freqs (first-most, second, third = p, q, r)
            counts = np.bincount(marr)

            pbase = np.argmax(counts)
            nump = counts[pbase]
            counts[pbase] = 0

            qbase = np.argmax(counts)
            numq = counts[qbase]
            counts[qbase] = 0

            rbase = np.argmax(counts)
            numr = counts[rbase]

            ## based on biallelic depth
            bidepth = nump + numq
            if bidepth < mindepth_majrule:
                cons[col] = 78

            else:
                ## if depth is too high, reduce to sampled int
                if bidepth > 500:
                    base1 = int(500 * (nump / float(bidepth)))
                    base2 = int(500 * (numq / float(bidepth)))
                else:
                    base1 = nump
                    base2 = numq

                ## make statistical base call
                if bidepth >= mindepth_statistical:
                    ishet, prob = get_binom(base1, base2, estE, estH)
                    #LOGGER.info("ishet, prob, b1, b2: %s %s %s %s", ishet, prob, base1, base2)
                    if prob < 0.95:
                        cons[col] = 78
                    else:
                        if ishet:
                            cons[col] = TRANS[(pbase, qbase)]
                        else:
                            cons[col] = pbase

                ## make majrule base call
                else: #if bidepth >= mindepth_majrule:
                    if nump == numq:
                        cons[col] = TRANS[(pbase, qbase)]
                    else:
                        cons[col] = pbase
    return cons.view("S1")


Instruction:
Given the following Python 3 function, write the documentation
def nfilter1(data, reps):
    if sum(reps) >= data.paramsdict["mindepth_majrule"] and \
        sum(reps) <= data.paramsdict["maxdepth"]:
        return 1
    else:
        return 0

Response: applies read depths filter


Instruction:
How would you explain what the following Python 3 function does
def nfilter4(consens, hidx, arrayed):

    ## if less than two Hs then there is only one allele
    if len(hidx) < 2:
        return consens, 1

    ## store base calls for hetero sites
    harray = arrayed[:, hidx]

    ## remove any reads that have N or - base calls at hetero sites
    ## these cannot be used when calling alleles currently.
    harray = harray[~np.any(harray == "-", axis=1)]
    harray = harray[~np.any(harray == "N", axis=1)]

    ## get counts of each allele (e.g., AT:2, CG:2)
    ccx = Counter([tuple(i) for i in harray])

    ## Two possibilities we would like to distinguish, but we can't. Therefore,
    ## we just throw away low depth third alleles that are within seq. error.
    ## 1) a third base came up as a sequencing error but is not a unique allele
    ## 2) a third or more unique allele is there but at low frequency

    ## remove low freq alleles if more than 2, since they may reflect
    ## sequencing errors at hetero sites, making a third allele, or a new
    ## allelic combination that is not real.
    if len(ccx) > 2:
        totdepth = harray.shape[0]
        cutoff = max(1, totdepth // 10)
        alleles = [i for i in ccx if ccx[i] > cutoff]
    else:
        alleles = ccx.keys()

    ## how many high depth alleles?
    nalleles = len(alleles)

    ## if 2 alleles then save the phase using lowercase coding
    if nalleles == 2:
        try:
            consens = storealleles(consens, hidx, alleles)
        except (IndexError, KeyError):
            ## the H sites do not form good alleles
            LOGGER.info("failed at phasing loc, skipping")
            LOGGER.info("""
    consens %s
    hidx %s
    alleles %s
                """, consens, hidx, alleles)
        return consens, nalleles
    ## just return the info for later filtering
    else:
        return consens, nalleles

Response: apply max haplotypes filter returns pass and consens


Instruction:
Can you generate the documentation for the following Python 3 function
def storealleles(consens, hidx, alleles):
    ## find the first hetero site and choose the priority base
    ## example, if W: then priority base in A and not T. PRIORITY=(order: CATG)
    bigbase = PRIORITY[consens[hidx[0]]]

    ## find which allele has priority based on bigbase
    bigallele = [i for i in alleles if i[0] == bigbase][0]

    ## uplow other bases relative to this one and the priority list
    ## e.g., if there are two hetero sites (WY) and the two alleles are
    ## AT and TC, then since bigbase of (W) is A second hetero site should
    ## be stored as y, since the ordering is swapped in this case; the priority
    ## base (C versus T) is C, but C goes with the minor base at h site 1.
    #consens = list(consens)
    for hsite, pbase in zip(hidx[1:], bigallele[1:]):
        if PRIORITY[consens[hsite]] != pbase:
            consens[hsite] = consens[hsite].lower()

    ## return consens
    return consens

Response: store phased allele data for diploids


Instruction:
Can you generate the documentation for the following Python 3 function
def cleanup(data, sample, statsdicts):
    LOGGER.info("in cleanup for: %s", sample.name)
    isref = 'reference' in data.paramsdict["assembly_method"]

    ## collect consens chunk files
    combs1 = glob.glob(os.path.join(
                        data.dirs.consens,
                        sample.name+"_tmpcons.*"))
    combs1.sort(key=lambda x: int(x.split(".")[-1]))

    ## collect tmpcat files
    tmpcats = glob.glob(os.path.join(
                        data.dirs.consens,
                        sample.name+"_tmpcats.*"))
    tmpcats.sort(key=lambda x: int(x.split(".")[-1]))

    ## get shape info from the first cat, (optim, maxlen, 4)
    with h5py.File(tmpcats[0], 'r') as io5:
        optim, maxlen, _ = io5['cats'].shape

    ## save as a chunked compressed hdf5 array
    handle1 = os.path.join(data.dirs.consens, sample.name+".catg")
    with h5py.File(handle1, 'w') as ioh5:
        nloci = len(tmpcats) * optim
        dcat = ioh5.create_dataset("catg", (nloci, maxlen, 4),
                                   dtype=np.uint32,
                                   chunks=(optim, maxlen, 4),
                                   compression="gzip")
        dall = ioh5.create_dataset("nalleles", (nloci, ),
                                   dtype=np.uint8,
                                   chunks=(optim, ),
                                   compression="gzip")
        ## only create chrom for reference-aligned data
        if isref:
            dchrom = ioh5.create_dataset("chroms", (nloci, 3),
                                         dtype=np.int64,
                                         chunks=(optim, 3),
                                         compression="gzip")

        ## Combine all those tmp cats into the big cat
        start = 0
        for icat in tmpcats:
            io5 = h5py.File(icat, 'r')
            end = start + optim
            dcat[start:end] = io5['cats'][:]
            dall[start:end] = io5['alls'][:]
            if isref:
                dchrom[start:end] = io5['chroms'][:]
            start += optim
            io5.close()
            os.remove(icat)

    ## store the handle to the Sample
    sample.files.database = handle1

    ## record results
    xcounters = {"nconsens": 0,
                 "heteros": 0,
                 "nsites": 0}
    xfilters = {"depth": 0,
               "maxh": 0,
               "maxn": 0}

    ## merge finished consens stats
    for counters, filters in statsdicts:
        ## sum individual counters
        for key in xcounters:
            xcounters[key] += counters[key]
        for key in xfilters:
            xfilters[key] += filters[key]

    ## merge consens read files
    handle1 = os.path.join(data.dirs.consens, sample.name+".consens.gz")
    with gzip.open(handle1, 'wb') as out:
        for fname in combs1:
            with open(fname) as infile:
                out.write(infile.read()+"\n")
            os.remove(fname)
    sample.files.consens = [handle1]

    ## set Sample stats_dfs values
    if int(xcounters['nsites']):
        prop = int(xcounters["heteros"]) / float(xcounters['nsites'])
    else:
        prop = 0

    sample.stats_dfs.s5.nsites = int(xcounters["nsites"])
    sample.stats_dfs.s5.nhetero = int(xcounters["heteros"])
    sample.stats_dfs.s5.filtered_by_depth = xfilters['depth']
    sample.stats_dfs.s5.filtered_by_maxH = xfilters['maxh']
    sample.stats_dfs.s5.filtered_by_maxN = xfilters['maxn']
    sample.stats_dfs.s5.reads_consens = int(xcounters["nconsens"])
    sample.stats_dfs.s5.clusters_total = sample.stats_dfs.s3.clusters_total
    sample.stats_dfs.s5.heterozygosity = float(prop)

    ## set the Sample stats summary value
    sample.stats.reads_consens = int(xcounters["nconsens"])

    ## save state to Sample if successful
    if sample.stats.reads_consens:
        sample.stats.state = 5
    else:
        print("No clusters passed filtering in Sample: {}".format(sample.name))
    return sample

Response: cleanup the unaligned data structures


Instruction:
Can you generate the documentation for the following Python 3 function
def chunk_clusters(data, sample):

    ## counter for split job submission
    num = 0

    ## set optim size for chunks in N clusters. The first few chunks take longer
    ## because they contain larger clusters, so we create 4X as many chunks as
    ## processors so that they are split more evenly.
    optim = int((sample.stats.clusters_total // data.cpus) + \
                (sample.stats.clusters_total % data.cpus))

    ## break up the file into smaller tmp files for each engine
    ## chunking by cluster is a bit trickier than chunking by N lines
    chunkslist = []

    ## open to clusters
    with gzip.open(sample.files.clusters, 'rb') as clusters:
        ## create iterator to sample 2 lines at a time
        pairdealer = itertools.izip(*[iter(clusters)]*2)

        ## Use iterator to sample til end of cluster
        done = 0
        while not done:
            ## grab optim clusters and write to file.
            done, chunk = clustdealer(pairdealer, optim)
            chunkhandle = os.path.join(data.dirs.clusts,
                                    "tmp_"+str(sample.name)+"."+str(num*optim))
            if chunk:
                chunkslist.append((optim, chunkhandle))
                with open(chunkhandle, 'wb') as outchunk:
                    outchunk.write("//\n//\n".join(chunk)+"//\n//\n")
                num += 1

    return chunkslist

Response: chunk the cluster file into smaller files


Instruction:
Can you generate the documentation for the following Python 3 function
def get_subsamples(data, samples, force):

    subsamples = []
    for sample in samples:
        if not force:
            if sample.stats.state >= 5:
                print("""\
    Skipping Sample {}; Already has consens reads. Use force arg to overwrite.\
    """.format(sample.name))
            elif not sample.stats.clusters_hidepth:
                print("""\
    Skipping Sample {}; No clusters found."""\
    .format(sample.name, int(sample.stats.clusters_hidepth)))
            elif sample.stats.state < 4:
                print("""\
    Skipping Sample {}; not yet finished step4 """\
    .format(sample.name))
            else:
                subsamples.append(sample)

        else:
            if not sample.stats.clusters_hidepth:
                print("""\
    Skipping Sample {}; No clusters found in {}."""\
    .format(sample.name, sample.files.clusters))
            elif sample.stats.state < 4:
                print("""\
    Skipping Sample {}; not yet finished step4"""\
    .format(sample.name))
            else:
                subsamples.append(sample)

    if len(subsamples) == 0:
        raise IPyradWarningExit("""
    No samples to cluster, exiting.
    """)

    ## if sample is already done skip
    if "hetero_est" not in data.stats:
        print("  No estimates of heterozygosity and error rate. Using default "\
              "values")
        for sample in subsamples:
            sample.stats.hetero_est = 0.001
            sample.stats.error_est = 0.0001

    if data._headers:
        print(u"""\
  Mean error  [{:.5f} sd={:.5f}]
  Mean hetero [{:.5f} sd={:.5f}]"""\
  .format(data.stats.error_est.mean(), data.stats.error_est.std(),
          data.stats.hetero_est.mean(), data.stats.hetero_est.std()))

    return subsamples

Response: Get the subsamples of the specified samples.


Instruction:
How would you explain what the following Python 3 function does
def run(data, samples, force, ipyclient):
    ## prepare dirs
    data.dirs.consens = os.path.join(data.dirs.project, data.name+"_consens")
    if not os.path.exists(data.dirs.consens):
        os.mkdir(data.dirs.consens)

    ## zap any tmp files that might be leftover
    tmpcons = glob.glob(os.path.join(data.dirs.consens, "*_tmpcons.*"))
    tmpcats = glob.glob(os.path.join(data.dirs.consens, "*_tmpcats.*"))
    for tmpfile in tmpcons+tmpcats:
        os.remove(tmpfile)

    ## filter through samples for those ready
    samples = get_subsamples(data, samples, force)

    ## set up parallel client: how many cores?
    lbview = ipyclient.load_balanced_view()
    data.cpus = data._ipcluster["cores"]
    if not data.cpus:
        data.cpus = len(ipyclient.ids)

    ## wrap everything to ensure destruction of temp files
    inst = ""
    try:
        ## calculate depths, if they changed.
        samples = calculate_depths(data, samples, lbview)

        ## chunk clusters into bits for parallel processing
        lasyncs = make_chunks(data, samples, lbview)

        ## process chunks and cleanup
        process_chunks(data, samples, lasyncs, lbview)

    except KeyboardInterrupt as inst:
        raise inst

    finally:
        ## if process failed at any point delete tmp files
        tmpcons = glob.glob(os.path.join(data.dirs.clusts, "tmp_*.[0-9]*"))
        tmpcons += glob.glob(os.path.join(data.dirs.consens, "*_tmpcons.*"))
        tmpcons += glob.glob(os.path.join(data.dirs.consens, "*_tmpcats.*"))
        for tmpchunk in tmpcons:
            os.remove(tmpchunk)

        ## Finished step 5. Set step 6 checkpoint to 0 to force
        ## re-running from scratch.
        data._checkpoint = 0

Response: run the sample and return the base class


Instruction:
Can you create a Python 3 function that
calculates the depths of the cluster by running the lbview function on each of the samples.

Response:
def calculate_depths(data, samples, lbview):
    """
    check whether mindepth has changed, and thus whether clusters_hidepth
    needs to be recalculated, and get new maxlen for new highdepth clusts.
    if mindepth not changed then nothing changes.
    """

    ## send jobs to be processed on engines
    start = time.time()
    printstr = " calculating depths    | {} | s5 |"
    recaljobs = {}
    maxlens = []
    for sample in samples:
        recaljobs[sample.name] = lbview.apply(recal_hidepth, *(data, sample))

    ## block until finished
    while 1:
        ready = [i.ready() for i in recaljobs.values()]
        elapsed = datetime.timedelta(seconds=int(time.time()-start))
        progressbar(len(ready), sum(ready), printstr.format(elapsed), spacer=data._spacer)
        time.sleep(0.1)
        if len(ready) == sum(ready):
            print("")
            break

    ## check for failures and collect results
    modsamples = []
    for sample in samples:
        if not recaljobs[sample.name].successful():
            LOGGER.error("  sample %s failed: %s", sample.name, recaljobs[sample.name].exception())
        else:
            modsample, _, maxlen, _, _ = recaljobs[sample.name].result()
            modsamples.append(modsample)
            maxlens.append(maxlen)

    ## reset global maxlen if something changed
    data._hackersonly["max_fragment_length"] = int(max(maxlens)) + 4

    return samples


Instruction:
How would you implement a function in Python 3 that
makes a chunk of the data into a single node

Response:
def make_chunks(data, samples, lbview):
    """
    calls chunk_clusters and tracks progress.
    """
    ## first progress bar
    start = time.time()
    printstr = " chunking clusters     | {} | s5 |"
    elapsed = datetime.timedelta(seconds=int(time.time()-start))
    progressbar(10, 0, printstr.format(elapsed), spacer=data._spacer)

    ## send off samples to be chunked
    lasyncs = {}
    for sample in samples:
        lasyncs[sample.name] = lbview.apply(chunk_clusters, *(data, sample))

    ## block until finished
    while 1:
        ready = [i.ready() for i in lasyncs.values()]
        elapsed = datetime.timedelta(seconds=int(time.time()-start))
        progressbar(len(ready), sum(ready), printstr.format(elapsed), spacer=data._spacer)
        time.sleep(0.1)
        if len(ready) == sum(ready):
            print("")
            break

    ## check for failures
    for sample in samples:
        if not lasyncs[sample.name].successful():
            LOGGER.error("  sample %s failed: %s", sample.name,
                        lasyncs[sample.name].exception())

    return lasyncs


Instruction:
Given the following Python 3 function, write the documentation
def process_chunks(data, samples, lasyncs, lbview):

    ## send chunks to be processed
    start = time.time()
    asyncs = {sample.name:[] for sample in samples}
    printstr = " consens calling       | {} | s5 |"

    ## get chunklist from results
    for sample in samples:
        clist = lasyncs[sample.name].result()
        for optim, chunkhandle in clist:
            args = (data, sample, chunkhandle, optim)
            #asyncs[sample.name].append(lbview.apply_async(consensus, *args))
            asyncs[sample.name].append(lbview.apply_async(newconsensus, *args))
            elapsed = datetime.timedelta(seconds=int(time.time()-start))
            progressbar(10, 0, printstr.format(elapsed), spacer=data._spacer)

    ## track progress
    allsyncs = list(itertools.chain(*[asyncs[i.name] for i in samples]))
    while 1:
        ready = [i.ready() for i in allsyncs]
        elapsed = datetime.timedelta(seconds=int(time.time()-start))
        progressbar(len(ready), sum(ready), printstr.format(elapsed), spacer=data._spacer)
        time.sleep(0.1)
        if len(ready) == sum(ready):
            break

    ## get clean samples
    casyncs = {}
    for sample in samples:
        rlist = asyncs[sample.name]
        statsdicts = [i.result() for i in rlist]
        casyncs[sample.name] = lbview.apply(cleanup, *(data, sample, statsdicts))
    while 1:
        ready = [i.ready() for i in casyncs.values()]
        elapsed = datetime.timedelta(seconds=int(time.time()-start))
        progressbar(10, 10, printstr.format(elapsed), spacer=data._spacer)
        time.sleep(0.1)
        if len(ready) == sum(ready):
            print("")
            break

    ## check for failures:
    for key in asyncs:
        asynclist = asyncs[key]
        for async in asynclist:
            if not async.successful():
                LOGGER.error("  async error: %s \n%s", key, async.exception())
    for key in casyncs:
        if not casyncs[key].successful():
            LOGGER.error("  casync error: %s \n%s", key, casyncs[key].exception())

    ## get samples back
    subsamples = [i.result() for i in casyncs.values()]
    for sample in subsamples:
        data.samples[sample.name] = sample

    ## build Assembly stats
    data.stats_dfs.s5 = data._build_stat("s5")

    ## write stats file
    data.stats_files.s5 = os.path.join(data.dirs.consens, 's5_consens_stats.txt')
    with io.open(data.stats_files.s5, 'w') as out:
        #out.write(data.stats_dfs.s5.to_string())
        data.stats_dfs.s5.to_string(
            buf=out,
            formatters={
                'clusters_total':'{:.0f}'.format,
                'filtered_by_depth':'{:.0f}'.format,
                'filtered_by_maxH':'{:.0f}'.format,
                'filtered_by_maxN':'{:.0f}'.format,
                'reads_consens':'{:.0f}'.format,
                'nsites':'{:.0f}'.format,
                'nhetero':'{:.0f}'.format,
                'heterozygosity':'{:.5f}'.format
            })

Response: process chunks from consens func and...


Instruction:
Write a Python 3 function for
reading in. loci and builds alleles from case characters

Response:
def make(data, samples):
    """ reads in .loci and builds alleles from case characters """

    #read in loci file
    outfile = open(os.path.join(data.dirs.outfiles, data.name+".alleles"), 'w')
    lines = open(os.path.join(data.dirs.outfiles, data.name+".loci"), 'r')

    ## Get the longest sample name for pretty printing
    longname = max(len(x) for x in data.samples.keys())

    ## Padding between name and sequence in output file. This should be the
    ## same as write_outfiles.write_tmp_loci.name_padding
    name_padding = 5
    writing = []
    loc = 0
    for line in lines:
        if ">" in line:
            name, seq = line.split(" ")[0], line.split(" ")[-1]
            allele1, allele2 = splitalleles(seq.strip())

            ## Format the output string. the "-2" below accounts for the additional
            ## 2 characters added to the sample name that don't get added to the
            ## snpsites line, so you gotta bump this line back 2 to make it
            ## line up right.
            writing.append(name+"_0"+" "*(longname-len(name)-2+name_padding)+allele1)
            writing.append(name+"_1"+" "*(longname-len(name)-2+name_padding)+allele2)
        else:
            writing.append(line.strip())
        loc += 1

        ## print every 10K loci "
        if not loc % 10000:
            outfile.write("\n".join(writing)+"\n")
            writing = []

    outfile.write("\n".join(writing))
    outfile.close()


Instruction:
Write a Python 3 script to
build SNPs and Stats files for a single sample.

Response:
def make(data, samples):
    """ builds snps output """

    ## get attr
    ploidy = data.paramsdict["max_alleles_consens"]
    names = [i.name for i in samples]
    longname = max([len(i) for i in names])

    ## TODO: use iter cuz of super huge files
    inloci = open(os.path.join(\
                  data.dirs.outfiles, data.name+".loci" ), 'r').read()

    ## Potential outfiles
    snpsout = os.path.join(data.dirs.outfiles, data.name+".snps")
    usnpsout = os.path.join(data.dirs.outfiles, data.name+".usnps")
    structout = os.path.join(data.dirs.outfiles, data.name+".str")
    genoout = os.path.join(data.dirs.outfiles, data.name+".snps.geno")
    ugenoout = os.path.join(data.dirs.outfiles, data.name+".usnps.geno")

    ## Output file for writing some stats
    statsfile= os.path.join(data.dirs.outfiles, data.name+".snps.stats")

    ## The output formats to write
    formats = data.paramsdict["output_formats"]

    seed = data._hackersonly["random_seed"]
    np.random.seed(int(seed))

    ## output .snps and .unlinked_snps"
    S = {}      ## snp dict
    Si = {}     ## unlinked snp dict
    for name in list(names):
        S[name] = []
        Si[name] = []

    ## record bi-allelic snps"
    bis = 0

    ## for each locus select out the SNPs"
    for loc in inloci.strip().split("|")[:-1]:
        pis = ""
        ns = []
        ss = []
        cov = {}  ## record coverage for each SNP
        for line in loc.split("\n"):
            if ">" in line:
                ns.append(line.split()[0].replace(">",""))
                ss.append(line.split()[-1])
            else:
                pis = [i[0] for i in enumerate(line) if i[1] in list('*-')]

        ## assign snps to S, and record coverage for usnps"
        for tax in S:
            if tax in ns:
                if pis:
                    for snpsite in pis:
                        snpsite -= (longname+5)
                        S[tax].append(ss[ns.index(tax)][snpsite])
                        if snpsite not in cov:
                            cov[snpsite] = 1
                        else:
                            cov[snpsite] += 1
                        ## downweight selection of gap sites "
                        if ss[ns.index(tax)][snpsite] != '-':
                           cov[snpsite] += 1
            else:
                if pis:
                    for snpsite in pis:
                        S[tax].append("N")
                    Si[tax].append("N")

        ## randomly select among snps w/ greatest coverage for unlinked snp "
        maxlist = []
        for j,k in cov.items():
            if k == max(cov.values()):
                maxlist.append(j)

        ## Is bi-allelic ? "
        bisnps = []
        for i in maxlist:
            if len(set([ss[ns.index(tax)][i] for tax in S if tax in ns])) < 3:
                bisnps.append(i)

        #rando = pis[np.random.randint(len(pis))]
        #rando -= (longname+5)
        if bisnps:
            rando = bisnps[np.random.randint(len(bisnps))]
        elif maxlist:
            rando = maxlist[np.random.randint(len(maxlist))]
        tbi = 0
        for tax in S:
            if tax in ns:
                if pis:
                    ## if none are bi-allelic "
                    if not bisnps:
                        tbi += 1
                    Si[tax].append(ss[ns.index(tax)][rando])
            if pis:
                ## add spacer between loci "
                S[tax].append(" ")
            else:
                ## invariable locus "
                S[tax].append("_ ")
        bis += tbi
    ## names
    SF = list(S.keys())
    SF.sort()

    ## Write linked snps format
    if "snps" in formats:
        with open(snpsout, 'w') as outfile:
            print >>outfile, "## %s taxa, %s loci, %s snps" % \
            (len(S), len("".join(S.values()[0]).split(" "))-1,
                    len("".join(S[SF[0]]).replace(" ", "")))
            for i in SF:
                print >>outfile, i+(" "*(longname-len(i)+3))+"".join(S[i])

    ## Write unlinked snps format
    if "usnps" in formats:
        with open(usnpsout, 'w') as outfile:
            print >>outfile, len(Si), len("".join(Si.values()[0]))
            for i in SF:
                print >>outfile, i+(" "*(longname-len(i)+3))+"".join(Si[i])

        with open(statsfile, 'a') as statsout:
            print >>statsout, "sampled unlinked SNPs=",len(Si.values()[0])
            print >>statsout, "sampled unlinked bi-allelic SNPs=",len(Si.values()[0])-bis

    ## Write STRUCTURE format
    if "str" in formats:
        with open(structout, 'w') as outfile:

            B = {'A': '0',
                 'T': '1',
                 'G': '2',
                 'C': '3',
                 'N': '-9',
                 '-': '-9'}
            if ploidy > 1:
                for line in SF:
                    print >>outfile, line+(" "*(longname-len(line)+3))+\
                          "\t"*6+"\t".join([B[unstruct(j)[0]] for j in Si[line]])
                    print >>outfile, line+(" "*(longname-len(line)+3))+\
                          "\t"*6+"\t".join([B[unstruct(j)[1]] for j in Si[line]])
            else:
                for line in SF:
                    print >>outfile, line+(" "*(longname-len(line)+3))+\
                          "\t"*6+"\t".join([B[unstruct(j)[1]] for j in Si[line]])

    ## Do linked and unlinked snps in .geno format
    if "geno" in formats:
        with open(ugenoout, 'w') as outfile:

            for i in range(len(Si.values()[0])):
                getref = 0
                ref = "N"
                while ref == "N":
                    ref = unstruct(Si[SF[getref]][i])[0]
                    getref += 1
                SNProw = "".join(map(str,[unstruct(Si[j][i]).count(ref) if Si[j][i] != "N" \
                                          else "9" for j in SF]))
                ## print ref,SNProw
                if len(set(SNProw)) > 1:
                    print >>outfile, SNProw

        with open(genoout, 'w') as outfile:
            for i in range(len(S.values()[0])):
                if S[SF[0]][i].strip("_").strip():
                    getref = 0
                    ref = "N"
                    while ref == "N":
                        #print i, S[SF[0]][i]
                        ref = unstruct(S[SF[getref]][i])[0]
                        getref += 1
                        SNProw = "".join(map(str,[unstruct(S[j][i]).count(ref) if \
                                                  S[j][i] != "N" else "9" for j in SF]))
                    ## print ref,SNProw
                    if len(set(SNProw)) > 1:
                        print >>outfile, SNProw


Instruction:
Explain what the following Python 3 code does
def cluster_info(ipyclient, spacer=""):
    ## get engine data, skips busy engines.
    hosts = []
    for eid in ipyclient.ids:
        engine = ipyclient[eid]
        if not engine.outstanding:
            hosts.append(engine.apply(_socket.gethostname))

    ## report it
    hosts = [i.get() for i in hosts]
    result = []
    for hostname in set(hosts):
        result.append("{}host compute node: [{} cores] on {}"\
            .format(spacer, hosts.count(hostname), hostname))
    print "\n".join(result)

Response: report host and engine info for an ipyclient


Instruction:
Here you have a function in Python 3, explain what it does
def _debug_on():
    ## make tmp file and set loglevel for top-level init
    with open(__debugflag__, 'w') as dfile:
        dfile.write("wat")
    __loglevel__ = "DEBUG"
    _LOGGER.info("debugging turned on and registered to be turned off at exit")
    _set_debug_dict(__loglevel__)

Response: Turn on debugging by creating hidden tmp file


Instruction:
Here you have a function in Python 3, explain what it does
def _set_debug_dict(__loglevel__):

    _lconfig.dictConfig({
    'version': 1,
    'disable_existing_loggers': False,

    'formatters': {
        'standard': {
            'format': "%(asctime)s \t"\
                     +"pid=%(process)d \t"\
                     +"[%(filename)s]\t"\
                     +"%(levelname)s \t"\
                     +"%(message)s"
        },
    },
    'handlers': {
        __name__: {
            'level':__loglevel__,
            'class':'logging.FileHandler',
            'filename':__debugfile__,
            'formatter':"standard",
            'mode':'a+'
        }
    },
    'loggers':{
        __name__: {
            'handlers': [__name__],
            'level': __loglevel__,
            'propogate': True
        }
    }
    })

Response: set the debug dict


Instruction:
How would you explain what the following Python 3 function does
def _debug_off():
    if _os.path.exists(__debugflag__):
        _os.remove(__debugflag__)
    __loglevel__ = "ERROR"
    _LOGGER.info("debugging turned off")
    _set_debug_dict(__loglevel__)

Response: Turn off debugging by removing hidden tmp file.


Instruction:
Write a Python 3 script to
check if dependency program exists

Response:
def _cmd_exists(cmd):
    """ check if dependency program is there """
    return _subprocess.call("type " + cmd,
                           shell=True,
                           stdout=_subprocess.PIPE,
                           stderr=_subprocess.PIPE) == 0


Instruction:
Explain what the following Python 3 code does
def _getbins():

    # Return error if system is 32-bit arch.
    # This is straight from the python docs:
    # https://docs.python.org/2/library/platform.html#cross-platform
    if not _sys.maxsize > 2**32:
        _sys.exit("ipyrad requires 64bit architecture")

    ## get platform mac or linux
    _platform = _sys.platform

    ## get current location
    if 'VIRTUAL_ENV' in _os.environ:
        ipyrad_path = _os.environ['VIRTUAL_ENV']
    else:
        path = _os.path.abspath(_os.path.dirname(__file__))
        ipyrad_path = _os.path.dirname(path)

    ## find bin directory
    ipyrad_path = _os.path.dirname(path)
    bin_path = _os.path.join(ipyrad_path, "bin")

    ## get the correct binaries
    if 'linux' in _platform:
        vsearch = _os.path.join(
                       _os.path.abspath(bin_path),
                       "vsearch-linux-x86_64")
        muscle = _os.path.join(
                       _os.path.abspath(bin_path),
                       "muscle-linux-x86_64")
        smalt = _os.path.join(
                       _os.path.abspath(bin_path),
                       "smalt-linux-x86_64")
        bwa = _os.path.join(
                       _os.path.abspath(bin_path),
                       "bwa-linux-x86_64")
        samtools = _os.path.join(
                       _os.path.abspath(bin_path),
                       "samtools-linux-x86_64")
        bedtools = _os.path.join(
                       _os.path.abspath(bin_path),
                       "bedtools-linux-x86_64")
        qmc = _os.path.join(
                       _os.path.abspath(bin_path),
                       "QMC-linux-x86_64")
    else:
        vsearch = _os.path.join(
                       _os.path.abspath(bin_path),
                       "vsearch-osx-x86_64")
        muscle = _os.path.join(
                       _os.path.abspath(bin_path),
                       "muscle-osx-x86_64")
        smalt = _os.path.join(
                       _os.path.abspath(bin_path),
                       "smalt-osx-x86_64")
        bwa = _os.path.join(
                       _os.path.abspath(bin_path),
                       "bwa-osx-x86_64")
        samtools = _os.path.join(
                       _os.path.abspath(bin_path),
                       "samtools-osx-x86_64")
        bedtools = _os.path.join(
                       _os.path.abspath(bin_path),
                       "bedtools-osx-x86_64")
        ## only one compiled version available, works for all?
        qmc = _os.path.join(
                       _os.path.abspath(bin_path),
                       "QMC-osx-x86_64")

    # Test for existence of binaries
    assert _cmd_exists(muscle), "muscle not found here: "+muscle
    assert _cmd_exists(vsearch), "vsearch not found here: "+vsearch
    assert _cmd_exists(smalt), "smalt not found here: "+smalt
    assert _cmd_exists(bwa), "bwa not found here: "+bwa
    assert _cmd_exists(samtools), "samtools not found here: "+samtools
    assert _cmd_exists(bedtools), "bedtools not found here: "+bedtools
    #assert _cmd_exists(qmc), "wQMC not found here: "+qmc
    return vsearch, muscle, smalt, bwa, samtools, bedtools, qmc

Response: Get the right version of vsearch muscle smalt bwa and qqtools depending on linux vs osx


Instruction:
How would you explain what the following Python 3 function does
def store_all(self):

    with h5py.File(self.database.input, 'a') as io5:
        fillsets = io5["quartets"]

        ## generator for all quartet sets
        qiter = itertools.combinations(xrange(len(self.samples)), 4)
        i = 0
        while i < self.params.nquartets:
            ## sample a chunk of the next ordered N set of quartets
            dat = np.array(list(itertools.islice(qiter, self._chunksize)))
            end = min(self.params.nquartets, dat.shape[0]+i)
            fillsets[i:end] = dat[:end-i]
            i += self._chunksize

            ## send progress update to stdout on engine
            print(min(i, self.params.nquartets))

Response: Populate array with all possible quartets. This allows us to update the database with a checkpoint.


Instruction:
Here you have a function in Python 3, explain what it does
def store_random(self):

    with h5py.File(self.database.input, 'a') as io5:
        fillsets = io5["quartets"]

        ## set generators
        qiter = itertools.combinations(xrange(len(self.samples)), 4)
        rand = np.arange(0, n_choose_k(len(self.samples), 4))
        np.random.shuffle(rand)
        rslice = rand[:self.params.nquartets]
        rss = np.sort(rslice)
        riter = iter(rss)
        del rand, rslice

        ## print progress update 1 to the engine stdout
        print(self._chunksize)

        ## set to store
        rando = riter.next()
        tmpr = np.zeros((self.params.nquartets, 4), dtype=np.uint16)
        tidx = 0
        while 1:
            try:
                for i, j in enumerate(qiter):
                    if i == rando:
                        tmpr[tidx] = j
                        tidx += 1
                        rando = riter.next()

                    ## print progress bar update to engine stdout
                    if not i % self._chunksize:
                        print(min(i, self.params.nquartets))

            except StopIteration:
                break
        ## store into database
        fillsets[:] = tmpr
        del tmpr

Response: Populate array with random quartets sampled from a generator and store them into the database.


Instruction:
Write a Python 3 function for
taking a tetrad class object and populates array with random integers with unique integers for each element in the tree.

Response:
def store_equal(self):
    """
    Takes a tetrad class object and populates array with random
    quartets sampled equally among splits of the tree so that
    deep splits are not overrepresented relative to rare splits,
    like those near the tips.
    """

    with h5py.File(self.database.input, 'a') as io5:
        fillsets = io5["quartets"]

        ## require guidetree
        if not os.path.exists(self.files.tree):
            raise IPyradWarningExit(
                "To use sampling method 'equal' requires a guidetree")
        tre = ete3.Tree(self.files.tree)
        tre.unroot()
        tre.resolve_polytomy(recursive=True)

        ## randomly sample internals splits
        splits = [([self.samples.index(z.name) for z in i],
                   [self.samples.index(z.name) for z in j]) \
                   for (i, j) in tre.get_edges()]

        ## only keep internal splits, not single tip edges
        splits = [i for i in splits if all([len(j) > 1 for j in i])]

        ## how many min quartets shoudl be equally sampled from each split
        squarts = self.params.nquartets // len(splits)

        ## keep track of how many iterators are saturable.
        saturable = 0

        ## turn each into an iterable split sampler
        ## if the nquartets for that split is small, then sample all,
        ## if it is big then make it a random sampler for that split.
        qiters = []

        ## iterate over splits sampling quartets evenly
        for idx, split in enumerate(splits):
            ## if small number at this split then sample all possible sets
            ## we will exhaust this quickly and then switch to random for
            ## the larger splits.
            total = n_choose_k(len(split[0]), 2) * n_choose_k(len(split[1]), 2)
            if total < squarts*2:
                qiter = (i+j for (i, j) in itertools.product(
                            itertools.combinations(split[0], 2),
                            itertools.combinations(split[1], 2)))
                saturable += 1

            ## else create random sampler across that split, this is slower
            ## because it can propose the same split repeatedly and so we
            ## have to check it against the 'sampled' set.
            else:
                qiter = (random_product(split[0], split[1]) for _ \
                         in xrange(self.params.nquartets))

            ## store all iterators into a list
            qiters.append((idx, qiter))

        ## create infinite cycler of qiters
        qitercycle = itertools.cycle(qiters)

        ## store visited quartets
        sampled = set()

        ## fill chunksize at a time
        i = 0
        empty = set()
        edge_targeted = 0
        random_targeted = 0

        ## keep filling quartets until nquartets are sampled.
        while i < self.params.nquartets:
            ## grab the next iterator
            cycle, qiter = qitercycle.next()

            ## sample from iterators, store sorted set.
            try:
                qrtsamp = tuple(sorted(qiter.next()))
                if qrtsamp not in sampled:
                    sampled.add(qrtsamp)
                    edge_targeted += 1
                    i += 1
                    ## print progress bar update to engine stdout
                    if not i % self._chunksize:
                        print(min(i, self.params.nquartets))

            except StopIteration:
                empty.add(cycle)
                if len(empty) == saturable:
                    break

        ## if array is not full then add random samples
        while i <= self.params.nquartets:
            newset = tuple(sorted(np.random.choice(
                range(len(self.samples)), 4, replace=False)))
            if newset not in sampled:
                sampled.add(newset)
                random_targeted += 1
                i += 1
                ## print progress bar update to engine stdout
                if not i % self._chunksize:
                    print(min(i, self.params.nquartets))

        ## store into database
        print(self.params.nquartets)
        fillsets[:] = np.array(tuple(sampled))
        del sampled


Instruction:
Implement a function in Python 3 to
return nsets unique random quartet sets sampled from n - choose - k without replacement combinations.

Response:
def random_combination(nsets, n, k):
    """
    Returns nsets unique random quartet sets sampled from
    n-choose-k without replacement combinations.
    """
    sets = set()
    while len(sets) < nsets:
        newset = tuple(sorted(np.random.choice(n, k, replace=False)))
        sets.add(newset)
    return tuple(sets)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def set_mkl_thread_limit(cores):
    if "linux" in sys.platform:
        mkl_rt = ctypes.CDLL('libmkl_rt.so')
    else:
        mkl_rt = ctypes.CDLL('libmkl_rt.dylib')
    oldlimit = mkl_rt.mkl_get_max_threads()
    mkl_rt.mkl_set_num_threads(ctypes.byref(ctypes.c_int(cores)))
    return oldlimit

Response: set mkl thread limit and return old value so we can reset when finished.


Instruction:
Create a Python 3 function to
get total number of quartets possible for a split

Response:
def get_total(tots, node):
    """ get total number of quartets possible for a split"""
    if (node.is_leaf() or node.is_root()):
        return 0
    else:
        ## Get counts on down edges.
        ## How to treat polytomies here?
        if len(node.children) > 2:
            down_r = node.children[0]
            down_l = node.children[1]
            for child in node.children[2:]:
                down_l += child
        else:
            down_r, down_l = node.children
        lendr = sum(1 for i in down_r.iter_leaves())
        lendl = sum(1 for i in down_l.iter_leaves())

        ## get count on up edge sister
        up_r = node.get_sisters()[0]
        lenur = sum(1 for i in up_r.iter_leaves())

        ## everyone else
        lenul = tots - (lendr + lendl + lenur)

        ## return product
        return lendr * lendl * lenur * lenul