Instruction:
Can you generate a brief explanation for the following Python 3 code
def cleanup_tempfiles(data):

    ## remove align-related tmp files
    tmps1 = glob.glob(os.path.join(data.tmpdir, "*.fa"))
    tmps2 = glob.glob(os.path.join(data.tmpdir, "*.npy"))
    for tmp in tmps1 + tmps2:
        if os.path.exists(tmp):
            os.remove(tmp)

    ## remove cluster related files
    removal = [
        os.path.join(data.dirs.across, data.name+".utemp"),
        os.path.join(data.dirs.across, data.name+".htemp"),
        os.path.join(data.dirs.across, data.name+"_catcons.tmp"),
        os.path.join(data.dirs.across, data.name+"_cathaps.tmp"),
        os.path.join(data.dirs.across, data.name+"_catshuf.tmp"),
        os.path.join(data.dirs.across, data.name+"_catsort.tmp"),
        os.path.join(data.dirs.across, data.name+".tmparrs.h5"),
        os.path.join(data.dirs.across, data.name+".tmp.indels.hdf5"),
        ]
    for rfile in removal:
        if os.path.exists(rfile):
            os.remove(rfile)

    ## remove singlecat related h5 files
    smpios = glob.glob(os.path.join(data.dirs.across, '*.tmp.h5'))
    for smpio in smpios:
        if os.path.exists(smpio):
            os.remove(smpio)

Response: Function to remove old files from the temp files of the current sequence.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def assembly_cleanup(data):

    ## build s2 results data frame
    data.stats_dfs.s2 = data._build_stat("s2")
    data.stats_files.s2 = os.path.join(data.dirs.edits, 's2_rawedit_stats.txt')

    ## write stats for all samples
    with io.open(data.stats_files.s2, 'w', encoding='utf-8') as outfile:
        data.stats_dfs.s2.fillna(value=0).astype(np.int).to_string(outfile)

Response: cleanup for assembly object


Instruction:
Write a Python 3 script to
parse results from cutadapt into sample data

Response:
def parse_single_results(data, sample, res1):
    """ parse results from cutadapt into sample data"""

    ## set default values
    #sample.stats_dfs.s2["reads_raw"] = 0
    sample.stats_dfs.s2["trim_adapter_bp_read1"] = 0
    sample.stats_dfs.s2["trim_quality_bp_read1"] = 0
    sample.stats_dfs.s2["reads_filtered_by_Ns"] = 0
    sample.stats_dfs.s2["reads_filtered_by_minlen"] = 0
    sample.stats_dfs.s2["reads_passed_filter"] = 0

    ## parse new values from cutadapt results output
    lines = res1.strip().split("\n")
    for line in lines:

        if "Total reads processed:" in line:
            value = int(line.split()[3].replace(",", ""))
            sample.stats_dfs.s2["reads_raw"] = value

        if "Reads with adapters:" in line:
            value = int(line.split()[3].replace(",", ""))
            sample.stats_dfs.s2["trim_adapter_bp_read1"] = value

        if "Quality-trimmed" in line:
            value = int(line.split()[1].replace(",", ""))
            sample.stats_dfs.s2["trim_quality_bp_read1"] = value

        if "Reads that were too short" in line:
            value = int(line.split()[5].replace(",", ""))
            sample.stats_dfs.s2["reads_filtered_by_minlen"] = value

        if "Reads with too many N" in line:
            value = int(line.split()[5].replace(",", ""))
            sample.stats_dfs.s2["reads_filtered_by_Ns"] = value

        if "Reads written (passing filters):" in line:
            value = int(line.split()[4].replace(",", ""))
            sample.stats_dfs.s2["reads_passed_filter"] = value

    ## save to stats summary
    if sample.stats_dfs.s2.reads_passed_filter:
        sample.stats.state = 2
        sample.stats.reads_passed_filter = sample.stats_dfs.s2.reads_passed_filter
        sample.files.edits = [
            (OPJ(data.dirs.edits, sample.name+".trimmed_R1_.fastq.gz"), 0)]
        ## write the long form output to the log file.
        LOGGER.info(res1)

    else:
        print("{}No reads passed filtering in Sample: {}".format(data._spacer, sample.name))


Instruction:
Write a Python 3 script to
parse pair mod results from cutadapt

Response:
def parse_pair_results(data, sample, res):
    """ parse results from cutadapt for paired data"""
    LOGGER.info("in parse pair mod results\n%s", res)
    ## set default values
    sample.stats_dfs.s2["trim_adapter_bp_read1"] = 0
    sample.stats_dfs.s2["trim_adapter_bp_read2"] = 0
    sample.stats_dfs.s2["trim_quality_bp_read1"] = 0
    sample.stats_dfs.s2["trim_quality_bp_read2"] = 0
    sample.stats_dfs.s2["reads_filtered_by_Ns"] = 0
    sample.stats_dfs.s2["reads_filtered_by_minlen"] = 0
    sample.stats_dfs.s2["reads_passed_filter"] = 0

    lines = res.strip().split("\n")
    qprimed = 0
    for line in lines:
        ## set primer to catch next line
        if "Quality-trimmed" in line:
            qprimed = 1

        ## grab read1 and read2 lines when qprimed
        if "Read 1:" in line:
            if qprimed:
                value = int(line.split()[2].replace(",", ""))
                sample.stats_dfs.s2["trim_quality_bp_read1"] = value

        if "Read 2:" in line:
            if qprimed:
                value = int(line.split()[2].replace(",", ""))
                sample.stats_dfs.s2["trim_quality_bp_read2"] = value
                qprimed = 0

        if "Read 1 with adapter:" in line:
            value = int(line.split()[4].replace(",", ""))
            sample.stats_dfs.s2["trim_adapter_bp_read1"] = value

        if "Read 2 with adapter:" in line:
            value = int(line.split()[4].replace(",", ""))
            sample.stats_dfs.s2["trim_adapter_bp_read2"] = value

        if "Total read pairs processed:" in line:
            value = int(line.split()[4].replace(",", ""))
            sample.stats_dfs.s2["reads_raw"] = value

        if "Pairs that were too short" in line:
            value = int(line.split()[5].replace(",", ""))
            sample.stats_dfs.s2["reads_filtered_by_minlen"] = value

        if "Pairs with too many N" in line:
            value = int(line.split()[5].replace(",", ""))
            sample.stats_dfs.s2["reads_filtered_by_Ns"] = value

        if "Pairs written (passing filters):" in line:
            value = int(line.split()[4].replace(",", ""))
            sample.stats_dfs.s2["reads_passed_filter"] = value

    ## save to stats summary
    if sample.stats_dfs.s2.reads_passed_filter:
        sample.stats.state = 2
        sample.stats.reads_passed_filter = sample.stats_dfs.s2.reads_passed_filter
        sample.files.edits = [(
             OPJ(data.dirs.edits, sample.name+".trimmed_R1_.fastq.gz"),
             OPJ(data.dirs.edits, sample.name+".trimmed_R2_.fastq.gz")
             )]

    else:
        print("No reads passed filtering in Sample: {}".format(sample.name))


Instruction:
Write a Python 3 script for
applying quality and adapter filters to reads using cutadapt. If the ipyrad filter param is set to 0 then it only filters to hard trim edges and uses mintrimlen. If filter=1, we add quality filters. If filter=2 we add adapter filters.

Response:
def cutadaptit_single(data, sample):
    """
    Applies quality and adapter filters to reads using cutadapt. If the ipyrad
    filter param is set to 0 then it only filters to hard trim edges and uses
    mintrimlen. If filter=1, we add quality filters. If filter=2 we add
    adapter filters.
    """

    sname = sample.name
    ## if (GBS, ddRAD) we look for the second cut site + adapter. For single-end
    ## data we don't bother trying to remove the second barcode since it's not
    ## as critical as with PE data.
    if data.paramsdict["datatype"] == "rad":
        adapter = data._hackersonly["p3_adapter"]
    else:
        ## if GBS then the barcode can also be on the other side.
        if data.paramsdict["datatype"] == "gbs":

            ## make full adapter (-revcompcut-revcompbarcode-adapter)
            ## and add adapter without revcompbarcode
            if data.barcodes:
                adapter = \
                    fullcomp(data.paramsdict["restriction_overhang"][1])[::-1] \
                  + fullcomp(data.barcodes[sample.name])[::-1] \
                  + data._hackersonly["p3_adapter"]
                ## add incomplete adapter to extras (-recompcut-adapter)
                data._hackersonly["p3_adapters_extra"].append(
                    fullcomp(data.paramsdict["restriction_overhang"][1])[::-1] \
                  + data._hackersonly["p3_adapter"])
            else:
                LOGGER.warning("No barcode information present, and is therefore not "+\
                               "being used for adapter trimming of SE gbs data.")
                ## else no search for barcodes on 3'
                adapter = \
                    fullcomp(data.paramsdict["restriction_overhang"][1])[::-1] \
                  + data._hackersonly["p3_adapter"]
        else:
            adapter = \
                fullcomp(data.paramsdict["restriction_overhang"][1])[::-1] \
              + data._hackersonly["p3_adapter"]

    ## get length trim parameter from new or older version of ipyrad params
    trim5r1 = trim3r1 = []
    if data.paramsdict.get("trim_reads"):
        trimlen = data.paramsdict.get("trim_reads")

        ## trim 5' end
        if trimlen[0]:
            trim5r1 = ["-u", str(trimlen[0])]
        if trimlen[1] < 0:
            trim3r1 = ["-u", str(trimlen[1])]
        if trimlen[1] > 0:
            trim3r1 = ["--length", str(trimlen[1])]
    else:
        trimlen = data.paramsdict.get("edit_cutsites")
        trim5r1 = ["--cut", str(trimlen[0])]

    ## testing new 'trim_reads' setting
    cmdf1 = ["cutadapt"]
    if trim5r1:
        cmdf1 += trim5r1
    if trim3r1:
        cmdf1 += trim3r1
    cmdf1 += ["--minimum-length", str(data.paramsdict["filter_min_trim_len"]),
              "--max-n", str(data.paramsdict["max_low_qual_bases"]),
              "--trim-n",
              "--output", OPJ(data.dirs.edits, sname+".trimmed_R1_.fastq.gz"),
              sample.files.concat[0][0]]

    if int(data.paramsdict["filter_adapters"]):
        ## NEW: only quality trim the 3' end for SE data.
        cmdf1.insert(1, "20")
        cmdf1.insert(1, "-q")
        cmdf1.insert(1, str(data.paramsdict["phred_Qscore_offset"]))
        cmdf1.insert(1, "--quality-base")

    ## if filter_adapters==3 then p3_adapters_extra will already have extra
    ## poly adapters added to its list.
    if int(data.paramsdict["filter_adapters"]) > 1:
        ## first enter extra cuts (order of input is reversed)
        for extracut in list(set(data._hackersonly["p3_adapters_extra"]))[::-1]:
            cmdf1.insert(1, extracut)
            cmdf1.insert(1, "-a")
        ## then put the main cut so it appears first in command
        cmdf1.insert(1, adapter)
        cmdf1.insert(1, "-a")

    ## do modifications to read1 and write to tmp file
    LOGGER.info(cmdf1)
    proc1 = sps.Popen(cmdf1, stderr=sps.STDOUT, stdout=sps.PIPE, close_fds=True)
    try:
        res1 = proc1.communicate()[0]
    except KeyboardInterrupt:
        proc1.kill()
        raise KeyboardInterrupt

    ## raise errors if found
    if proc1.returncode:
        raise IPyradWarningExit(" error in {}\n {}".format(" ".join(cmdf1), res1))

    ## return result string to be parsed outside of engine
    return res1


Instruction:
Can you generate a brief explanation for the following Python 3 code
def cutadaptit_pairs(data, sample):
    LOGGER.debug("Entering cutadaptit_pairs - {}".format(sample.name))
    sname = sample.name

    ## applied to read pairs
    #trim_r1 = str(data.paramsdict["edit_cutsites"][0])
    #trim_r2 = str(data.paramsdict["edit_cutsites"][1])
    finput_r1 = sample.files.concat[0][0]
    finput_r2 = sample.files.concat[0][1]

    ## Get adapter sequences. This is very important. For the forward adapter
    ## we don't care all that much about getting the sequence just before the
    ## Illumina adapter, b/c it will either be random (in RAD), or the reverse
    ## cut site of cut1 or cut2 (gbs or ddrad). Either way, we can still trim it
    ## off later in step7 with trim overhang if we want. And it should be invar-
    ## iable unless the cut site has an ambiguous char. The reverse adapter is
    ## super important, however b/c it can contain the inline barcode and
    ## revcomp cut site. We def want to trim out the barcode, and ideally the
    ## cut site too to be safe. Problem is we don't always know the barcode if
    ## users demultiplexed their data elsewhere. So, if barcode is missing we
    ## do a very fuzzy match before the adapter and trim it out.

    ## this just got more complicated now that we allow merging technical
    ## replicates in step 1 since a single sample might have multiple barcodes
    ## associated with it and so we need to search for multiple adapter+barcode
    ## combinations.
    ## We will assume that if they are 'linking_barcodes()' here then there are
    ## no technical replicates in the barcodes file. If there ARE technical
    ## replicates, then they should run step1 so they are merged, in which case
    ## the sample specific barcodes will be saved to each Sample under its
    ## .barcode attribute as a list.

    if not data.barcodes:
        ## try linking barcodes again in case user just added a barcodes path
        ## after receiving the warning. We assume no technical replicates here.
        try:
            data._link_barcodes()
        except Exception as inst:
            LOGGER.warning("  error adding barcodes info: %s", inst)

    ## barcodes are present meaning they were parsed to the samples in step 1.
    if data.barcodes:
        try:
            adapter1 = fullcomp(data.paramsdict["restriction_overhang"][1])[::-1] \
                        + data._hackersonly["p3_adapter"]
            if isinstance(sample.barcode, list):
                bcode = fullcomp(sample.barcode[0])[::-1]
            elif isinstance(data.barcodes[sample.name], list):
                bcode = fullcomp(data.barcodes[sample.name][0][::-1])
            else:
                bcode = fullcomp(data.barcodes[sample.name])[::-1]
            ## add full adapter (-revcompcut-revcompbcode-adapter)
            adapter2 = fullcomp(data.paramsdict["restriction_overhang"][0])[::-1] \
                        + bcode \
                        + data._hackersonly["p5_adapter"]
        except KeyError as inst:
            msg = """
    Sample name does not exist in the barcode file. The name in the barcode file
    for each sample must exactly equal the raw file name for the sample minus
    `_R1`. So for example a sample called WatDo_PipPrep_R1_100.fq.gz must
    be referenced in the barcode file as WatDo_PipPrep_100. The name in your
    barcode file for this sample must match: {}
    """.format(sample.name)
            LOGGER.error(msg)
            raise IPyradWarningExit(msg)
    else:
        print(NO_BARS_GBS_WARNING)
        #adapter1 = fullcomp(data.paramsdict["restriction_overhang"][1])[::-1]+\
        #           data._hackersonly["p3_adapter"]
        #adapter2 = "XXX"
        adapter1 = data._hackersonly["p3_adapter"]
        adapter2 = fullcomp(data._hackersonly["p5_adapter"])

    ## parse trim_reads
    trim5r1 = trim5r2 = trim3r1 = trim3r2 = []
    if data.paramsdict.get("trim_reads"):
        trimlen = data.paramsdict.get("trim_reads")

        ## trim 5' end
        if trimlen[0]:
            trim5r1 = ["-u", str(trimlen[0])]
        if trimlen[1] < 0:
            trim3r1 = ["-u", str(trimlen[1])]
        if trimlen[1] > 0:
            trim3r1 = ["--length", str(trimlen[1])]

        ## legacy support for trimlen = 0,0 default
        if len(trimlen) > 2:
            if trimlen[2]:
                trim5r2 = ["-U", str(trimlen[2])]

        if len(trimlen) > 3:
            if trimlen[3]:
                if trimlen[3] < 0:
                    trim3r2 = ["-U", str(trimlen[3])]
                if trimlen[3] > 0:
                    trim3r2 = ["--length", str(trimlen[3])]

    else:
        ## legacy support
        trimlen = data.paramsdict.get("edit_cutsites")
        trim5r1 = ["-u", str(trimlen[0])]
        trim5r2 = ["-U", str(trimlen[1])]

    ## testing new 'trim_reads' setting
    cmdf1 = ["cutadapt"]
    if trim5r1:
        cmdf1 += trim5r1
    if trim3r1:
        cmdf1 += trim3r1
    if trim5r2:
        cmdf1 += trim5r2
    if trim3r2:
        cmdf1 += trim3r2

    cmdf1 += ["--trim-n",
              "--max-n", str(data.paramsdict["max_low_qual_bases"]),
              "--minimum-length", str(data.paramsdict["filter_min_trim_len"]),
              "-o", OPJ(data.dirs.edits, sname+".trimmed_R1_.fastq.gz"),
              "-p", OPJ(data.dirs.edits, sname+".trimmed_R2_.fastq.gz"),
              finput_r1,
              finput_r2]

    ## additional args
    if int(data.paramsdict["filter_adapters"]) < 2:
        ## add a dummy adapter to let cutadapt know whe are not using legacy-mode
        cmdf1.insert(1, "XXX")
        cmdf1.insert(1, "-A")

    if int(data.paramsdict["filter_adapters"]):
        cmdf1.insert(1, "20,20")
        cmdf1.insert(1, "-q")
        cmdf1.insert(1, str(data.paramsdict["phred_Qscore_offset"]))
        cmdf1.insert(1, "--quality-base")

    if int(data.paramsdict["filter_adapters"]) > 1:
        ## if technical replicates then add other copies
        if isinstance(sample.barcode, list):
            for extrabar in sample.barcode[1:]:
                data._hackersonly["p5_adapters_extra"] += \
                    fullcomp(data.paramsdict["restriction_overhang"][0])[::-1] + \
                    fullcomp(extrabar)[::-1] + \
                    data._hackersonly["p5_adapter"]
                data._hackersonly["p5_adapters_extra"] += \
                    fullcomp(data.paramsdict["restriction_overhang"][1])[::-1] + \
                    data._hackersonly["p3_adapter"]

        ## first enter extra cuts
        zcut1 = list(set(data._hackersonly["p3_adapters_extra"]))[::-1]
        zcut2 = list(set(data._hackersonly["p5_adapters_extra"]))[::-1]
        for ecut1, ecut2 in zip(zcut1, zcut2):
            cmdf1.insert(1, ecut1)
            cmdf1.insert(1, "-a")
            cmdf1.insert(1, ecut2)
            cmdf1.insert(1, "-A")
        ## then put the main cut first
        cmdf1.insert(1, adapter1)
        cmdf1.insert(1, '-a')
        cmdf1.insert(1, adapter2)
        cmdf1.insert(1, '-A')

    ## do modifications to read1 and write to tmp file
    LOGGER.debug(" ".join(cmdf1))
    #sys.exit()
    try:
        proc1 = sps.Popen(cmdf1, stderr=sps.STDOUT, stdout=sps.PIPE, close_fds=True)
        res1 = proc1.communicate()[0]
    except KeyboardInterrupt:
        proc1.kill()
        LOGGER.info("this is where I want it to interrupt")
        raise KeyboardInterrupt()

    ## raise errors if found
    if proc1.returncode:
        raise IPyradWarningExit(" error [returncode={}]: {}\n{}"\
            .format(proc1.returncode, " ".join(cmdf1), res1))

    LOGGER.debug("Exiting cutadaptit_pairs - {}".format(sname))
    ## return results string to be parsed outside of engine
    return res1

Response: Given a sample and a list of reads that are in a single sample we can use that to find the adapter sequences that are in the pair of reads.


Instruction:
Make a summary of the following Python 3 code
def run2(data, samples, force, ipyclient):

    ## create output directories
    data.dirs.edits = os.path.join(os.path.realpath(
                                   data.paramsdict["project_dir"]),
                                   data.name+"_edits")
    if not os.path.exists(data.dirs.edits):
        os.makedirs(data.dirs.edits)

    ## get samples
    subsamples = choose_samples(samples, force)

    ## only allow extra adapters in filters==3,
    ## and add poly repeats if not in list of adapters
    if int(data.paramsdict["filter_adapters"]) == 3:
        if not data._hackersonly["p3_adapters_extra"]:
            for poly in ["A"*8, "T"*8, "C"*8, "G"*8]:
                data._hackersonly["p3_adapters_extra"].append(poly)
        if not data._hackersonly["p5_adapters_extra"]:
            for poly in ["A"*8, "T"*8, "C"*8, "G"*8]:
                data._hackersonly["p5_adapters_extra"].append(poly)
    else:
        data._hackersonly["p5_adapters_extra"] = []
        data._hackersonly["p3_adapters_extra"] = []

    ## concat is not parallelized (since it's disk limited, generally)
    subsamples = concat_reads(data, subsamples, ipyclient)

    ## cutadapt is parallelized by ncores/2 because cutadapt spawns threads
    lbview = ipyclient.load_balanced_view(targets=ipyclient.ids[::2])
    run_cutadapt(data, subsamples, lbview)

    ## cleanup is ...
    assembly_cleanup(data)

Response: Run cutadapt on the given samples.


Instruction:
Can you create a Python 3 function that
concatenates if multiple input files for a single samples

Response:
def concat_reads(data, subsamples, ipyclient):
    """ concatenate if multiple input files for a single samples """

    ## concatenate reads if they come from merged assemblies.
    if any([len(i.files.fastqs) > 1 for i in subsamples]):
        ## run on single engine for now
        start = time.time()
        printstr = " concatenating inputs  | {} | s2 |"
        finished = 0
        catjobs = {}
        for sample in subsamples:
            if len(sample.files.fastqs) > 1:
                catjobs[sample.name] = ipyclient[0].apply(\
                                       concat_multiple_inputs, *(data, sample))
            else:
                sample.files.concat = sample.files.fastqs

        ## wait for all to finish
        while 1:
            finished = sum([i.ready() for i in catjobs.values()])
            elapsed = datetime.timedelta(seconds=int(time.time()-start))
            progressbar(len(catjobs), finished, printstr.format(elapsed), spacer=data._spacer)
            time.sleep(0.1)
            if finished == len(catjobs):
                print("")
                break

        ## collect results, which are concat file handles.
        for async in catjobs:
            if catjobs[async].successful():
                data.samples[async].files.concat = catjobs[async].result()
            else:
                error = catjobs[async].result()#exception()
                LOGGER.error("error in step2 concat %s", error)
                raise IPyradWarningExit("error in step2 concat: {}".format(error))
    else:
        for sample in subsamples:
            ## just copy fastqs handles to concat attribute
            sample.files.concat = sample.files.fastqs

    return subsamples


Instruction:
Implement a Python 3 function for
running cutadapt on the subsamples and return the list of subsamples

Response:
def run_cutadapt(data, subsamples, lbview):
    """
    sends fastq files to cutadapt
    """
    ## choose cutadapt function based on datatype
    start = time.time()
    printstr = " processing reads      | {} | s2 |"
    finished = 0
    rawedits = {}

    ## sort subsamples so that the biggest files get submitted first
    subsamples.sort(key=lambda x: x.stats.reads_raw, reverse=True)
    LOGGER.info([i.stats.reads_raw for i in subsamples])

    ## send samples to cutadapt filtering
    if "pair" in data.paramsdict["datatype"]:
        for sample in subsamples:
            rawedits[sample.name] = lbview.apply(cutadaptit_pairs, *(data, sample))
    else:
        for sample in subsamples:
            rawedits[sample.name] = lbview.apply(cutadaptit_single, *(data, sample))

    ## wait for all to finish
    while 1:
        finished = sum([i.ready() for i in rawedits.values()])
        elapsed = datetime.timedelta(seconds=int(time.time()-start))
        progressbar(len(rawedits), finished, printstr.format(elapsed), spacer=data._spacer)
        time.sleep(0.1)
        if finished == len(rawedits):
            print("")
            break

    ## collect results, report failures, and store stats. async = sample.name
    for async in rawedits:
        if rawedits[async].successful():
            res = rawedits[async].result()

            ## if single cleanup is easy
            if "pair" not in data.paramsdict["datatype"]:
                parse_single_results(data, data.samples[async], res)
            else:
                parse_pair_results(data, data.samples[async], res)
        else:
            print("  found an error in step2; see ipyrad_log.txt")
            LOGGER.error("error in run_cutadapt(): %s", rawedits[async].exception())


Instruction:
Given the following Python 3 function, write the documentation
def choose_samples(samples, force):

    ## hold samples that pass
    subsamples = []
    ## filter the samples again
    if not force:
        for sample in samples:
            if sample.stats.state >= 2:
                print("""\
    Skipping Sample {}; Already filtered. Use force argument to overwrite.\
    """.format(sample.name))
            elif not sample.stats.reads_raw:
                print("""\
    Skipping Sample {}; No reads found in file {}\
    """.format(sample.name, sample.files.fastqs))
            else:
                subsamples.append(sample)

    else:
        for sample in samples:
            if not sample.stats.reads_raw:
                print("""\
    Skipping Sample {}; No reads found in file {}\
    """.format(sample.name, sample.files.fastqs))
            else:
                subsamples.append(sample)
    return subsamples

Response: Choose samples from the sequence of samples.


Instruction:
Create a Python 3 function to
concatenate multiple fastq files into one.

Response:
def concat_multiple_inputs(data, sample):
    """
    If multiple fastq files were appended into the list of fastqs for samples
    then we merge them here before proceeding.
    """

    ## if more than one tuple in fastq list
    if len(sample.files.fastqs) > 1:
        ## create a cat command to append them all (doesn't matter if they
        ## are gzipped, cat still works). Grab index 0 of tuples for R1s.
        cmd1 = ["cat"] + [i[0] for i in sample.files.fastqs]

        isgzip = ".gz"
        if not sample.files.fastqs[0][0].endswith(".gz"):
            isgzip = ""

        ## write to new concat handle
        conc1 = os.path.join(data.dirs.edits, sample.name+"_R1_concat.fq{}".format(isgzip))
        with open(conc1, 'w') as cout1:
            proc1 = sps.Popen(cmd1, stderr=sps.STDOUT, stdout=cout1, close_fds=True)
            res1 = proc1.communicate()[0]
        if proc1.returncode:
            raise IPyradWarningExit("error in: {}, {}".format(cmd1, res1))

        ## Only set conc2 if R2 actually exists
        conc2 = 0
        if "pair" in data.paramsdict["datatype"]:
            cmd2 = ["cat"] + [i[1] for i in sample.files.fastqs]
            conc2 = os.path.join(data.dirs.edits, sample.name+"_R2_concat.fq{}".format(isgzip))
            with open(conc2, 'w') as cout2:
                proc2 = sps.Popen(cmd2, stderr=sps.STDOUT, stdout=cout2, close_fds=True)
                res2 = proc2.communicate()[0]
            if proc2.returncode:
                raise IPyradWarningExit("Error concatenating fastq files. Make sure all "\
                    + "these files exist: {}\nError message: {}".format(cmd2, proc2.returncode))

        ## store new file handles
        sample.files.concat = [(conc1, conc2)]
    return sample.files.concat


Instruction:
Can you generate a brief explanation for the following Python 3 code
def make( data, samples ):
    invcffile   =  os.path.join( data.dirs.consens, data.name+".vcf" )
    outlocifile  =  os.path.join( data.dirs.outfiles, data.name+".loci" )

    importvcf( invcffile, outlocifile )

Response: Convert from step6 to. loci format


Instruction:
Write a Python 3 function for
functioning for importing a vcf file into loci format.

Response:
def importvcf( vcffile, locifile ):
    """ Function for importing a vcf file into loci format. Arguments
        are the input vcffile and the loci file to write out. """

    try:
        ## Get names of all individuals in the vcf
        with open( invcffile, 'r' ) as invcf:
            for line in invcf:
                if line.split()[0] == "#CHROM":
                    ## This is maybe a little clever. The names in the vcf are everything after
                    ## the "FORMAT" column, so find that index, then slice everything after it.
                    names_col = line.split().index( "FORMAT" ) + 1
                    names = line.split()[ names_col:]
                    LOGGER.debug( "Got names - %s", names )
                    break

            print( "wat" )
        ## Get the column to start reading at
    except Exception:
        print( "wat" )


Instruction:
Can you tell what is the following Python 3 function doing
def get_targets(ipyclient):
    ## fill hosts with async[gethostname]
    hosts = []
    for eid in ipyclient.ids:
        engine = ipyclient[eid]
        if not engine.outstanding:
            hosts.append(engine.apply(socket.gethostname))

    ## capture results of asyncs
    hosts = [i.get() for i in hosts]
    hostset = set(hosts)
    hostzip = zip(hosts, ipyclient.ids)
    hostdict = {host: [i[1] for i in hostzip if i[0] == host] for host in hostset}
    targets = list(itertools.chain(*[hostdict[i][:2] for i in hostdict]))

    ## return first two engines from each host
    return targets

Response: A function to find 2 engines per hostname on the ipyclient.


Instruction:
Write a Python 3 script for
computing stats for tree features and NHX tree features

Response:
def compute_tree_stats(self, ipyclient):
    """
    compute stats for stats file and NHX tree features
    """

    ## get name indices
    names = self.samples

    ## get majority rule consensus tree of weighted Q bootstrap trees
    if self.params.nboots:

        ## Tree object
        fulltre = ete3.Tree(self.trees.tree, format=0)
        fulltre.unroot()

        ## only grab as many boots as the last option said was max
        with open(self.trees.boots, 'r') as inboots:
            bb = [ete3.Tree(i.strip(), format=0) for i in inboots.readlines()]
            wboots = [fulltre] + bb[-self.params.nboots:]

        ## infer consensus tree and write to file
        wctre, wcounts = consensus_tree(wboots, names=names)
        self.trees.cons = os.path.join(self.dirs, self.name + ".cons")
        with open(self.trees.cons, 'w') as ocons:
            ocons.write(wctre.write(format=0))
    else:
        wctre = ete3.Tree(self.trees.tree, format=0)
        wctre.unroot()

    ## build stats file and write trees
    self.trees.nhx = os.path.join(self.dirs, self.name + ".nhx")
    with open(self.files.stats, 'w') as ostats:

        ## print Tetrad info
        #ostats.write(STATS_STRING.format(**self.stats))

        ## print bootstrap splits
        if self.params.nboots:
            ostats.write("## splits observed in {} trees\n".format(len(wboots)))
            for i, j in enumerate(self.samples):
                ostats.write("{:<3} {}\n".format(i, j))
            ostats.write("\n")
            for split, freq in wcounts:
                if split.count('1') > 1:
                    ostats.write("{}   {:.2f}\n".format(split, round(freq, 2)))
            ostats.write("\n")

    ## parallelized this function because it can be slogging
    lbview = ipyclient.load_balanced_view()

    ## store results in dicts
    qtots = {}
    qsamp = {}
    tots = sum(1 for i in wctre.iter_leaves())
    totn = set(wctre.get_leaf_names())

    ## iterate over node traversal.
    for node in wctre.traverse():
        ## this is slow, needs to look at every sampled quartet
        ## so we send it be processed on an engine
        qtots[node] = lbview.apply(_get_total, *(tots, node))
        qsamp[node] = lbview.apply(_get_sampled, *(self, totn, node))

    ## wait for jobs to finish
    ipyclient.wait()

    ## put results into tree
    for node in wctre.traverse():
        ## this is fast, just calcs n_choose_k
        total = qtots[node].result()
        sampled = qsamp[node].result()
        ## store the results to the tree
        node.add_feature("quartets_total", total)
        node.add_feature("quartets_sampled", sampled)
    features = ["quartets_total", "quartets_sampled"]

    ## return as NHX format with extra info
    with open(self.trees.nhx, 'w') as outtre:
        outtre.write(wctre.write(format=0, features=features))


Instruction:
Explain what the following Python 3 code does
def random_product(iter1, iter2):
    pool1 = tuple(iter1)
    pool2 = tuple(iter2)
    ind1 = random.sample(pool1, 2)
    ind2 = random.sample(pool2, 2)
    return tuple(ind1+ind2)

Response: random sampler for equal_splits func


Instruction:
Can you write a function in Python 3 where it
gets the number of quartets as n - choose - k

Response:
def n_choose_k(n, k):
    """ get the number of quartets as n-choose-k. This is used
    in equal splits to decide whether a split should be exhaustively sampled
    or randomly sampled. Edges near tips can be exhaustive while highly nested
    edges probably have too many quartets
    """
    return int(reduce(MUL, (Fraction(n-i, i+1) for i in range(k)), 1))


Instruction:
Can you generate the documentation for the following Python 3 function
def count_snps(mat):

    ## get [aabb, baba, abba, aaab]
    snps = np.zeros(4, dtype=np.uint32)

    ## get concordant (aabb) pis sites
    snps[0] = np.uint32(\
           mat[0, 5] + mat[0, 10] + mat[0, 15] + \
           mat[5, 0] + mat[5, 10] + mat[5, 15] + \
           mat[10, 0] + mat[10, 5] + mat[10, 15] + \
           mat[15, 0] + mat[15, 5] + mat[15, 10])

    ## get discordant (baba) sites
    for i in range(16):
        if i % 5:
            snps[1] += mat[i, i]

    ## get discordant (abba) sites
    snps[2] = mat[1, 4] + mat[2, 8] + mat[3, 12] +\
              mat[4, 1] + mat[6, 9] + mat[7, 13] +\
              mat[8, 2] + mat[9, 6] + mat[11, 14] +\
              mat[12, 3] + mat[13, 7] + mat[14, 11]

    ## get autapomorphy sites
    snps[3] = (mat.sum() - np.diag(mat).sum()) - snps[2]

    return snps

Response: count_snps returns a tuple of the number of snps in the count array


Instruction:
Can you implement a function in Python 3 that
removes ncolumns from snparray prior to matrix calculation, and subsamples 'linked' snps (those from the same RAD locus) such that for these four samples only 1 SNP per locus is kept. This information comes from the 'map' array (map file).

Response:
def subsample_snps_map(seqchunk, nmask, maparr):
    """
    removes ncolumns from snparray prior to matrix calculation, and
    subsamples 'linked' snps (those from the same RAD locus) such that
    for these four samples only 1 SNP per locus is kept. This information
    comes from the 'map' array (map file).
    """
    ## mask columns that contain Ns
    rmask = np.zeros(seqchunk.shape[1], dtype=np.bool_)

    ## apply mask to the mapfile
    last_loc = -1
    for idx in xrange(maparr.shape[0]):
        if maparr[idx] != last_loc:
            if not nmask[idx]:
                rmask[idx] = True
            last_loc = maparr[idx]

    ## apply mask
    #newarr = seqchunk[:, rmask]

    ## return smaller Nmasked array
    return rmask


Instruction:
Can you implement a function in Python 3 that
converts a chunk of numpy array into a list of numpy arrays

Response:
def chunk_to_matrices(narr, mapcol, nmask):
    """
    numba compiled code to get matrix fast.
    arr is a 4 x N seq matrix converted to np.int8
    I convert the numbers for ATGC into their respective index for the MAT
    matrix, and leave all others as high numbers, i.e., -==45, N==78.
    """

    ## get seq alignment and create an empty array for filling
    mats = np.zeros((3, 16, 16), dtype=np.uint32)

    ## replace ints with small ints that index their place in the
    ## 16x16. This no longer checks for big ints to exclude, so resolve=True
    ## is now the default, TODO.
    last_loc = -1
    for idx in xrange(mapcol.shape[0]):
        if not nmask[idx]:
            if not mapcol[idx] == last_loc:
                i = narr[:, idx]
                mats[0, (4*i[0])+i[1], (4*i[2])+i[3]] += 1
                last_loc = mapcol[idx]

    ## fill the alternates
    x = np.uint8(0)
    for y in np.array([0, 4, 8, 12], dtype=np.uint8):
        for z in np.array([0, 4, 8, 12], dtype=np.uint8):
            mats[1, y:y+np.uint8(4), z:z+np.uint8(4)] = mats[0, x].reshape(4, 4)
            mats[2, y:y+np.uint8(4), z:z+np.uint8(4)] = mats[0, x].reshape(4, 4).T
            x += np.uint8(1)

    return mats


Instruction:
Can you generate a brief explanation for the following Python 3 code
def calculate(seqnon, mapcol, nmask, tests):

    ## create empty matrices
    #LOGGER.info("tests[0] %s", tests[0])
    #LOGGER.info('seqnon[[tests[0]]] %s', seqnon[[tests[0]]])
    mats = chunk_to_matrices(seqnon, mapcol, nmask)

    ## empty svdscores for each arrangement of seqchunk
    svds = np.zeros((3, 16), dtype=np.float64)
    qscores = np.zeros(3, dtype=np.float64)
    ranks = np.zeros(3, dtype=np.float64)

    for test in range(3):
        ## get svd scores
        svds[test] = np.linalg.svd(mats[test].astype(np.float64))[1]
        ranks[test] = np.linalg.matrix_rank(mats[test].astype(np.float64))

    ## get minrank, or 11
    minrank = int(min(11, ranks.min()))
    for test in range(3):
        qscores[test] = np.sqrt(np.sum(svds[test, minrank:]**2))

    ## sort to find the best qorder
    best = np.where(qscores == qscores.min())[0]
    #best = qscores[qscores == qscores.min()][0]
    bidx = tests[best][0]
    qsnps = count_snps(mats[best][0])

    return bidx, qsnps

Response: calculate the best quality and qsnps for each seqnon


Instruction:
Here you have a function in Python 3, explain what it does
def nworker(data, smpchunk, tests):

    ## tell engines to limit threads
    #numba.config.NUMBA_DEFAULT_NUM_THREADS = 1

    ## open the seqarray view, the modified array is in bootsarr
    with h5py.File(data.database.input, 'r') as io5:
        seqview = io5["bootsarr"][:]
        maparr = io5["bootsmap"][:]

    ## create an N-mask array of all seq cols (this isn't really too slow)
    nall_mask = seqview[:] == 78

    ## tried numba compiling everythign below here, but was not faster
    ## than making nmask w/ axis arg in numpy

    ## get the input arrays ready
    rquartets = np.zeros((smpchunk.shape[0], 4), dtype=np.uint16)
    rweights = None
    #rweights = np.ones(smpchunk.shape[0], dtype=np.float64)
    rdstats = np.zeros((smpchunk.shape[0], 4), dtype=np.uint32)

    #times = []
    ## fill arrays with results using numba funcs
    for idx in xrange(smpchunk.shape[0]):
        ## get seqchunk for 4 samples (4, ncols)
        sidx = smpchunk[idx]
        seqchunk = seqview[sidx]

        ## get N-containing columns in 4-array, and invariant sites.
        nmask = np.any(nall_mask[sidx], axis=0)
        nmask += np.all(seqchunk == seqchunk[0], axis=0)  ## <- do we need this?

        ## get matrices if there are any shared SNPs
        ## returns best-tree index, qscores, and qstats
        #bidx, qscores, qstats = calculate(seqchunk, maparr[:, 0], nmask, tests)
        bidx, qstats = calculate(seqchunk, maparr[:, 0], nmask, tests)

        ## get weights from the three scores sorted.
        ## Only save to file if the quartet has information
        rdstats[idx] = qstats
        rquartets[idx] = smpchunk[idx][bidx]

    return rquartets, rweights, rdstats

Response: This function is used to workhorse the workhorse function.


Instruction:
Implement a function in Python 3 to
shuffle the columns of a sequence array

Response:
def shuffle_cols(seqarr, newarr, cols):
    """ used in bootstrap resampling without a map file """
    for idx in xrange(cols.shape[0]):
        newarr[:, idx] = seqarr[:, cols[idx]]
    return newarr


Instruction:
Can you create a Python 3 function that
returns a seq array with RSKWYM randomly replaced with resolved bases

Response:
def resolve_ambigs(tmpseq):
    """ returns a seq array with 'RSKYWM' randomly replaced with resolved bases"""
    ## iterate over the bases 'RSKWYM': [82, 83, 75, 87, 89, 77]
    for ambig in np.uint8([82, 83, 75, 87, 89, 77]):
        ## get all site in this ambig
        idx, idy = np.where(tmpseq == ambig)
        ## get the two resolutions of the ambig
        res1, res2 = AMBIGS[ambig.view("S1")]
        ## randomly sample half those sites
        halfmask = np.random.choice([True, False], idx.shape[0])
        ## replace ambig bases with their resolutions
        for i in xrange(halfmask.shape[0]):
            if halfmask[i]:
                tmpseq[idx[i], idy[i]] = np.array(res1).view(np.uint8)
            else:
                tmpseq[idx[i], idy[i]] = np.array(res2).view(np.uint8)
    return tmpseq


Instruction:
Create a Python 3 function to
get the span distance for each locus in original seqarray

Response:
def get_spans(maparr, spans):
    """ get span distance for each locus in original seqarray """
    ## start at 0, finds change at 1-index of map file
    bidx = 1
    spans = np.zeros((maparr[-1, 0], 2), np.uint64)
    ## read through marr and record when locus id changes
    for idx in xrange(1, maparr.shape[0]):
        cur = maparr[idx, 0]
        if cur != bidx:
            idy = idx + 1
            spans[cur-2, 1] = idx
            spans[cur-1, 0] = idx
            bidx = cur
    spans[-1, 1] = maparr[-1, -1]
    return spans


Instruction:
Can you generate the documentation for the following Python 3 function
def get_shape(spans, loci):
    width = 0
    for idx in xrange(loci.shape[0]):
        width += spans[loci[idx], 1] - spans[loci[idx], 0]
    return width

Response: get shape of new bootstrap resampled locus array


Instruction:
Can you generate the documentation for the following Python 3 function
def fill_boot(seqarr, newboot, newmap, spans, loci):
    ## column index
    cidx = 0

    ## resample each locus
    for i in xrange(loci.shape[0]):

        ## grab a random locus's columns
        x1 = spans[loci[i]][0]
        x2 = spans[loci[i]][1]
        cols = seqarr[:, x1:x2]

        ## randomize columns within colsq
        cord = np.random.choice(cols.shape[1], cols.shape[1], replace=False)
        rcols = cols[:, cord]

        ## fill bootarr with n columns from seqarr
        ## the required length was already measured
        newboot[:, cidx:cidx+cols.shape[1]] = rcols

        ## fill bootmap with new map info
        newmap[cidx: cidx+cols.shape[1], 0] = i+1

        ## advance column index
        cidx += cols.shape[1]

    ## return the concatenated cols
    return newboot, newmap

Response: fills the new boot array with the new map info


Instruction:
Can you generate the documentation for the following Python 3 function
def consensus_tree(trees, names=None, cutoff=0.0):

    ## find which clades occured with freq > cutoff
    namedict, clade_counts = _find_clades(trees, names=names)

    ## filter out the < cutoff clades
    fclade_counts = _filter_clades(clade_counts, cutoff)

    ## build tree
    consens_tree, _ = _build_trees(fclade_counts, namedict)
    ## make sure no singleton nodes were left behind
    return consens_tree, clade_counts

Response: This function is used to build a consensus tree from a list of trees.


Instruction:
Given the following Python 3 function, write the documentation
def refresh(self):

        ## clear any existing results files
        oldfiles = [self.files.qdump] + \
                    self.database.__dict__.values() + \
                    self.trees.__dict__.values()
        for oldfile in oldfiles:
            if oldfile:
                if os.path.exists(oldfile):
                    os.remove(oldfile)

        ## store old ipcluster info
        oldcluster = copy.deepcopy(self._ipcluster)

        ## reinit the tetrad object data.
        self.__init__(
            name=self.name,
            data=self.files.data,
            mapfile=self.files.mapfile,
            workdir=self.dirs,
            method=self.params.method,
            guidetreefile=self.files.guidetreefile,
            resolve=self._resolve,
            nboots=self.params.nboots,
            nquartets=self.params.nquartets,
            initarr=True,
            quiet=True,
            cli=self.kwargs.get("cli")
            )

        ## retain the same ipcluster info
        self._ipcluster = oldcluster

Response: Refresh the internal object with the current values of the object.


Instruction:
Create a Python 3 function for
parsing sample names from the sequence file

Response:
def _parse_names(self):
        """ parse sample names from the sequence file"""
        self.samples = []
        with iter(open(self.files.data, 'r')) as infile:
            infile.next().strip().split()
            while 1:
                try:
                    self.samples.append(infile.next().split()[0])
                except StopIteration:
                    break


Instruction:
How would you explain what the following Python 3 function does
def _init_seqarray(self, quiet=False):

        ## read in the data (seqfile)
        try:
            spath = open(self.files.data, 'r')
        except IOError:
            raise IPyradWarningExit(NO_SNP_FILE.format(self.files.data))
        line = spath.readline().strip().split()
        ntax = int(line[0])
        nbp = int(line[1])

        ## make a tmp seq array
        if not quiet:
            print("loading seq array [{} taxa x {} bp]".format(ntax, nbp))
        tmpseq = np.zeros((ntax, nbp), dtype=np.uint8)

        ## create array storage for real seq and the tmp bootstrap seqarray
        with h5py.File(self.database.input, 'w') as io5:
            io5.create_dataset("seqarr", (ntax, nbp), dtype=np.uint8)
            io5.create_dataset("bootsarr", (ntax, nbp), dtype=np.uint8)
            io5.create_dataset("bootsmap", (nbp, 2), dtype=np.uint32)

            ## if there is a map file, load it into the bootsmap
            if self.files.mapfile:
                with open(self.files.mapfile, 'r') as inmap:
                    ## parse the map file from txt and save as dataset
                    maparr = np.genfromtxt(inmap, dtype=np.uint64)
                    io5["bootsmap"][:] = maparr[:, [0, 3]]

                    ## parse the span info from maparr and save to dataset
                    spans = np.zeros((maparr[-1, 0], 2), np.uint64)
                    spans = get_spans(maparr, spans)
                    io5.create_dataset("spans", data=spans)
                    if not quiet:
                        print("max unlinked SNPs per quartet (nloci): {}"\
                              .format(spans.shape[0]))
            else:
                io5["bootsmap"][:, 0] = np.arange(io5["bootsmap"].shape[0])

            ## fill the tmp array from the input phy
            for line, seq in enumerate(spath.readlines()):
                tmpseq[line] = np.array(list(seq.split()[-1])).view(np.uint8)

            ## convert '-' or '_' into 'N'
            tmpseq[tmpseq == 45] = 78
            tmpseq[tmpseq == 95] = 78

            ## save array to disk so it can be easily accessed by slicing
            ## This unmodified array is used again later for sampling boots
            io5["seqarr"][:] = tmpseq

            ## resolve ambiguous IUPAC codes
            if self._resolve:
                tmpseq = resolve_ambigs(tmpseq)

            ## convert CATG bases to matrix indices
            tmpseq[tmpseq == 65] = 0
            tmpseq[tmpseq == 67] = 1
            tmpseq[tmpseq == 71] = 2
            tmpseq[tmpseq == 84] = 3

            ## save modified array to disk
            io5["bootsarr"][:] = tmpseq

            ## memory cleanup
            #del tmpseq

            ## get initial array
            LOGGER.info("original seqarr \n %s", io5["seqarr"][:, :20])
            LOGGER.info("original bootsarr \n %s", io5["bootsarr"][:, :20])
            LOGGER.info("original bootsmap \n %s", io5["bootsmap"][:20, :])

Response: Initialize the seqarray with the full data set and creates a bootstrapmap with the real data set.


Instruction:
Write a Python 3 script to
store the number of samples in a large array.

Response:
def _store_N_samples(self, ncpus):
        """
        Find all quartets of samples and store in a large array
        Create a chunk size for sampling from the array of quartets.
        This should be relatively large so that we don't spend a lot of time
        doing I/O, but small enough that jobs finish often for checkpointing.
        """
        breaks = 2
        if self.params.nquartets < 5000:
            breaks = 1
        if self.params.nquartets > 100000:
            breaks = 4
        if self.params.nquartets > 500000:
            breaks = 8

        ## chunk up the data
        self._chunksize = (self.params.nquartets // (breaks * ncpus) + \
                          (self.params.nquartets % (breaks * ncpus)))
        LOGGER.info("nquarts = %s, chunk = %s", self.params.nquartets, self._chunksize)

        ## 'samples' stores the indices of the quartet.
        ## `quartets` stores the correct quartet in the order (1,2|3,4)
        ## `weights` stores the weight of the quartet in 'quartets'
        ## we gzip this for now, but check later if this has a big speed cost

        ## create h5 OUT empty arrays
        with h5py.File(self.database.output, 'w') as io5:
            io5.create_dataset("quartets",
                               (self.params.nquartets, 4),
                               dtype=np.uint16,
                               chunks=(self._chunksize, 4))
            io5.create_dataset("qstats",
                               (self.params.nquartets, 4),
                               dtype=np.uint32,
                               chunks=(self._chunksize, 4))
            io5.create_group("qboots")

        ## append to h5 IN array (which also has seqarray) and fill it
        with h5py.File(self.database.input, 'a') as io5:
            ## create data sets
            io5.create_dataset("samples",
                               (self.params.nquartets, 4),
                               dtype=np.uint16,
                               chunks=(self._chunksize, 4),
                               compression='gzip')

            ## populate array with all possible quartets. This allows us to
            ## sample from the total, and also to continue from a checkpoint
            qiter = itertools.combinations(xrange(len(self.samples)), 4)
            i = 0

            ## fill chunksize at a time for efficiency
            while i < self.params.nquartets:
                if self.params.method != "all":
                    ## grab the next random 1000
                    qiter = []
                    while len(qiter) < min(self._chunksize, io5["samples"].shape[0]):
                        qiter.append(
                            random_combination(range(len(self.samples)), 4))
                    dat = np.array(qiter)
                else:
                    ## grab the next ordered chunksize
                    dat = np.array(list(itertools.islice(qiter, self._chunksize)))

                ## store to h5
                io5["samples"][i:i+self._chunksize] = dat[:io5["samples"].shape[0] - i]
                i += self._chunksize


Instruction:
Create a Python 3 function to
store the equal quartets in the database.

Response:
def _store_equal_samples(self, ncpus):
        """
        sample quartets evenly across splits of the starting tree, and fills
        in remaining samples with random quartet samples. Uses a hash dict to
        not sample the same quartet twice, so for very large trees this can
        take a few minutes to find millions of possible quartet samples.
        """

        ## choose chunker for h5 arr
        breaks = 2
        if self.params.nquartets < 5000:
            breaks = 1
        if self.params.nquartets > 100000:
            breaks = 4
        if self.params.nquartets > 500000:
            breaks = 8

        self._chunksize = (self.params.nquartets // (breaks * ncpus) + \
                         (self.params.nquartets % (breaks * ncpus)))
        LOGGER.info("nquarts = %s, chunk = %s", self.params.nquartets, self._chunksize)

        ## create h5 OUT empty arrays
        with h5py.File(self.database.output, 'w') as io5:
            io5.create_dataset("quartets",
                               (self.params.nquartets, 4),
                               dtype=np.uint16,
                               chunks=(self._chunksize, 4))
            io5.create_dataset("qstats",
                               (self.params.nquartets, 4),
                               dtype=np.uint32,
                               chunks=(self._chunksize, 4))
            io5.create_group("qboots")

        ## get starting tree, unroot, randomly resolve, ladderize
        tre = ete3.Tree(self.files.guidetreefile, format=0)
        #tre = toytree.tree(self.files.guidetreefile, format=0)
        tre.tree.unroot()
        tre.tree.resolve_polytomy(recursive=True)
        tre.tree.ladderize()

        ## randomly sample all splits of tree and convert tip names to indices
        splits = [([self.samples.index(z.name) for z in i],
                   [self.samples.index(z.name) for z in j]) \
                   for (i, j) in tre.get_edges()]

        ## only keep internal splits (no single tips edges)
        ## this seemed to cause problems with unsampled tips
        splits = [i for i in splits if all([len(j) > 1 for j in i])]

        ## turn each into an iterable split sampler
        ## if the nquartets for that split is small, then sample all of them
        ## if it is big, then make it a random sampler from that split
        qiters = []

        ## how many min quartets are we gonna sample from each split?
        squarts = self.params.nquartets // len(splits)

        ## how many iterators can be sampled to saturation?
        nsaturation = 0

        for split in splits:
            ## if small number at this split then sample all possible sets
            ## we will exhaust this quickly and then switch to random for
            ## the larger splits.
            if n_choose_k(len(split[0]), 2) * n_choose_k(len(split[1]), 2) < squarts*2:
                qiter = (i+j for (i, j) in itertools.product(
                            itertools.combinations(split[0], 2),
                            itertools.combinations(split[1], 2)))
                nsaturation += 1

            ## else create random sampler across that split, this is slower
            ## because it can propose the same split repeatedly and so we
            ## have to check it against the 'sampled' set.
            else:
                qiter = (random_product(split[0], split[1]) for _ \
                         in xrange(self.params.nquartets))
                nsaturation += 1

            ## store all iterators into a list
            qiters.append(qiter)

        #for split in splits:
        #    print(split)

        ## make qiters infinitely cycling
        qiters = itertools.cycle(qiters)
        cycler = itertools.cycle(range(len(splits)))

        ## store visiting quartets
        sampled = set()

        ## iterate over qiters sampling from each, if one runs out, keep
        ## sampling from remaining qiters. Keep going until samples is filled
        with h5py.File(self.database.input, 'a') as io5:
            ## create data sets
            io5.create_dataset("samples",
                               (self.params.nquartets, 4),
                               dtype=np.uint16,
                               chunks=(self._chunksize, 4),
                               compression='gzip')

            ## fill chunksize at a time for efficiency
            i = 0
            empty = set()
            edge_targeted = 0
            random_target = 0

            ## keep filling quartets until nquartets are sampled
            while i < self.params.nquartets:
                qdat = []
                ## keep filling this chunk until its full
                while len(qdat) < self._chunksize:
                    ## grab the next iterator
                    qiter = qiters.next()
                    cycle = cycler.next()

                    ## sample from iterator
                    try:
                        qrtsamp = qiter.next()
                        if tuple(qrtsamp) not in sampled:
                            qdat.append(qrtsamp)
                            sampled.add(qrtsamp)
                            edge_targeted += 1
                        #else:
                        #    print('repeat')

                    ## unless iterator is empty, then skip it
                    except StopIteration:
                        empty.add(cycle)

                    ## break when all edge samplers are empty
                    if len(empty) == nsaturation:
                        break

                ## if array is not full then add random samples
                while len(qdat) < self._chunksize:
                    qrtsamp = random_combination(range(len(self.samples)), 4)
                    if tuple(qrtsamp) not in sampled:
                        qdat.append(qrtsamp)
                        sampled.add(qrtsamp)
                        random_target += 1

                ## stick chunk into h5 array
                dat = np.array(qdat, dtype=np.uint16)
                io5["samples"][i:i+self._chunksize] = dat[:io5["samples"].shape[0] - i]
                i += self._chunksize

            print("  equal sampling: {} edge quartets, {} random quartets "\
                  .format(edge_targeted, random_target))


Instruction:
Can you implement a function in Python 3 that
runs QMC and saves the result in self. trees. tree and self. boots.

Response:
def _run_qmc(self, boot):
        """ runs quartet max-cut on a quartets file """

        ## convert to txt file for wQMC
        self._tmp = os.path.join(self.dirs, ".tmpwtre")
        cmd = [ip.bins.qmc, "qrtt="+self.files.qdump, "otre="+self._tmp]

        ## run them
        proc = subprocess.Popen(cmd, stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
        res = proc.communicate()
        if proc.returncode:
            #LOGGER.error("Error in QMC: \n({}).".format(res))
            LOGGER.error(res)
            raise IPyradWarningExit(res[1])

        ## read in the tmp files since qmc does not pipe
        with open(self._tmp) as intree:
            ## convert int names back to str names renamer returns a newick str
            #tmp = toytree.tree(intree.read().strip())
            tmp = ete3.Tree(intree.read().strip())
            tmpwtre = self._renamer(tmp)#.tree)

        ## save the tree
        if boot:
            self.trees.boots = os.path.join(self.dirs, self.name+".boots")
            with open(self.trees.boots, 'a') as outboot:
                outboot.write(tmpwtre+"\n")
        else:
            self.trees.tree = os.path.join(self.dirs, self.name+".tree")
            with open(self.trees.tree, 'w') as outtree:
                outtree.write(tmpwtre)

        ## save JSON file checkpoint
        self._save()


Instruction:
Explain what the following Python 3 code does
def _dump_qmc(self):
        ## open the h5 database
        io5 = h5py.File(self.database.output, 'r')

        ## create an output file for writing
        self.files.qdump = os.path.join(self.dirs, self.name+".quartets.txt")
        LOGGER.info("qdump file %s", self.files.qdump)
        outfile = open(self.files.qdump, 'w')

        ## todo: should pull quarts order in randomly? or doesn't matter?
        for idx in xrange(0, self.params.nquartets, self._chunksize):
            ## get mask of zero weight quartets
            #mask = io5["weights"][idx:idx+self.chunksize] != 0
            #weight = io5["weights"][idx:idx+self.chunksize][mask]
            #LOGGER.info("exluded = %s, mask shape %s",
            #            self._chunksize - mask.shape[0], mask.shape)
            #LOGGER.info('q shape %s', io5["quartets"][idx:idx+self._chunksize].shape)
            masked_quartets = io5["quartets"][idx:idx+self._chunksize, :]#[mask, :]
            quarts = [list(j) for j in masked_quartets]

            ## format and print
            #chunk = ["{},{}|{},{}:{}".format(*i+[j]) for i, j \
            #                                        in zip(quarts, weight)]
            chunk = ["{},{}|{},{}".format(*i) for i in quarts]
            outfile.write("\n".join(chunk)+"\n")

        ## close output file and h5 database
        outfile.close()
        io5.close()

Response: Dumps the quartets and weights of the entry into a file.


Instruction:
Here you have a function in Python 3, explain what it does
def _renamer(self, tre):
        ## get the tre with numbered tree tip labels
        names = tre.get_leaves()

        ## replace numbered names with snames
        for name in names:
            name.name = self.samples[int(name.name)]

        ## return with only topology and leaf labels
        return tre.write(format=9)

Response: renames newick from numbers to sample names


Instruction:
How would you explain what the following Python 3 function does
def _finalize_stats(self, ipyclient):

        ## print stats file location:
        #print(STATSOUT.format(opr(self.files.stats)))

        ## print finished tree information ---------------------
        print(FINALTREES.format(opr(self.trees.tree)))

        ## print bootstrap information --------------------------
        if self.params.nboots:
            ## get consensus, map values to tree edges, record stats file
            self._compute_tree_stats(ipyclient)
            ## print bootstrap info
            print(BOOTTREES.format(opr(self.trees.cons), opr(self.trees.boots)))

        ## print the ASCII tree only if its small
        if len(self.samples) < 20:
            if self.params.nboots:
                wctre = ete3.Tree(self.trees.cons, format=0)
                wctre.ladderize()
                print(wctre.get_ascii(show_internal=True,
                                      attributes=["dist", "name"]))
                print("")
            else:
                qtre = ete3.Tree(self.trees.tree, format=0)
                qtre.ladderize()
                #qtre = toytree.tree(self.trees.tree, format=0)
                #qtre.tree.unroot()
                print(qtre.get_ascii())
                print("")

        ## print PDF filename & tips -----------------------------
        docslink = "https://toytree.readthedocs.io/"
        citelink = "https://ipyrad.readthedocs.io/tetrad.html"
        print(LINKS.format(docslink, citelink))

Response: Write final tree files and print final statistics.


Instruction:
Write a Python 3 function that can
save a JSON file representation of the Tetrad Class for checkpoint

Response:
def _save(self):
        """ save a JSON file representation of Tetrad Class for checkpoint"""

        ## save each attribute as dict
        fulldict = copy.deepcopy(self.__dict__)
        for i, j in fulldict.items():
            if isinstance(j, Params):
                fulldict[i] = j.__dict__
        fulldumps = json.dumps(fulldict,
                               sort_keys=False,
                               indent=4,
                               separators=(",", ":"),
                               )

        ## save to file, make dir if it wasn't made earlier
        assemblypath = os.path.join(self.dirs, self.name+".tet.json")
        if not os.path.exists(self.dirs):
            os.mkdir(self.dirs)

        ## protect save from interruption
        done = 0
        while not done:
            try:
                with open(assemblypath, 'w') as jout:
                    jout.write(fulldumps)
                done = 1
            except (KeyboardInterrupt, SystemExit):
                print('.')
                continue


Instruction:
Create a Python 3 function to
insert the results from workers into hdf4 array

Response:
def _insert_to_array(self, start, results):
        """ inputs results from workers into hdf4 array """
        qrts, wgts, qsts = results
        #qrts, wgts = results
        #print(qrts)

        with h5py.File(self.database.output, 'r+') as out:
            chunk = self._chunksize
            out['quartets'][start:start+chunk] = qrts
            ##out['weights'][start:start+chunk] = wgts

            ## entered as 0-indexed !
            if self.checkpoint.boots:
                key = "qboots/b{}".format(self.checkpoint.boots-1)
                out[key][start:start+chunk] = qsts
            else:
                out["qstats"][start:start+chunk] = qsts


Instruction:
Can you create a Python 3 function that
runs the quartet inference on a SNP alignment and distribute workon a single cluster.

Response:
def run(self, force=0, verbose=2, ipyclient=None):
        """
        Run quartet inference on a SNP alignment and distribute work
        across an ipyparallel cluster (ipyclient). Unless passed an
        ipyclient explicitly, it looks for a running ipcluster instance
        running from the defautl ("") profile, and will raise an exception
        if one is not found within a set time limit. If not using the default
        profile then you can set "profile" as an argument to the tetrad object.
        Parameter settings influencing the run (e.g., nquartets, method) should
        be set on the tetrad Class object itself.

        Parameters
        ----------
        force (bool):
            Overwrite results for an object with this name if they exist.
        verbose (int):
            0=print nothing, 1=print progress bars, 2=print progress bars and
            print cluster info.
        ipyclient (ipyparallel.Client object):
            Default is None (use running Default ipcluster instance). To use
            a different ipcluster instance start a Client class object
            and pass it in as an argument here.
        """

        ## clear object results and data if force=True
        if force:
            self.refresh()

        ## wrap everything in a try statement so we can ensure that it will
        ## save if interrupted and we will clean up the
        inst = None
        try:
            ## launch and connect to ipcluster instance if doesn't exist
            if not ipyclient:
                args = self._ipcluster.items() + [("spacer", "")]
                ipyclient = ip.core.parallel.get_client(**dict(args))

            ## print a message about the cluster status
            if verbose == 2:
                ip.cluster_info(ipyclient)

            ## grab 2 engines from each host (2 multi-thread jobs per host)
            ## skips over engines that are busy running something else to avoid
            ## blocking if user is sharing an ipcluster.
            targets = get_targets(ipyclient)
            lbview = ipyclient.load_balanced_view(targets=targets)

            ## store ipyclient pids to the ipcluster instance so we can
            ## hard-kill them later.
            self._ipcluster["pids"] = ipyclient[:].apply(os.getpid).get_dict()

            ## get or init quartet sampling ---------------------------
            ## if load=True then chunksize will exist and this will skip
            if not self._chunksize:
                #self.nquartets = n_choose_k(len(self.samples), 4)
                ## store N sampled quartets into the h5 array
                if self.params.method != 'equal':
                    self._store_N_samples(ncpus=len(lbview))
                else:
                    self._store_equal_samples(ncpus=len(lbview))

            ## calculate invariants for the full array ----------------
            start = time.time()
            if not self.trees.tree:
                if verbose:
                    print("inferring {} induced quartet trees".format(self.params.nquartets))
                self._inference(start, lbview, quiet=verbose == 0)
                if verbose:
                    print("")
            else:
                if verbose:
                    print("initial tree already inferred")

            ## calculate for bootstraps -------------------------------
            start = time.time()
            if self.params.nboots:
                if self.checkpoint.boots == self.params.nboots:
                    if verbose:
                        print("{} bootstrap trees already inferred".format(self.params.nboots))
                else:
                    while self.checkpoint.boots < self.params.nboots:
                        ## resample bootsstrap seqarray
                        if self.files.mapfile:
                            self._sample_bootseq_array_map()
                        else:
                            self._sample_bootseq_array()

                        ## start boot inference, (1-indexed !!!)
                        self.checkpoint.boots += 1
                        self._inference(start, lbview, quiet=verbose == 0)
                if verbose:
                    print("")

            ## write outputs with bootstraps ---------------------------
            self.files.stats = os.path.join(self.dirs, self.name+"_stats.txt")
            if not self.kwargs.get("cli"):
                self._compute_tree_stats(ipyclient)
            else:
                self._finalize_stats(ipyclient)

        ## handle exceptions so they will be raised after we clean up below
        except KeyboardInterrupt as inst:
            LOGGER.info("assembly interrupted by user.")
            print("\nKeyboard Interrupt by user. Cleaning up...")

        except IPyradWarningExit as inst:
            LOGGER.info("IPyradWarningExit: %s", inst)
            print(inst)

        except Exception as inst:
            LOGGER.info("caught an unknown exception %s", inst)
            print("\n  Exception found: {}".format(inst))

        ## close client when done or interrupted
        finally:
            try:
                ## save the Assembly
                self._save()

                ## can't close client if it was never open
                if ipyclient:

                    ## send SIGINT (2) to all engines
                    ipyclient.abort()
                    LOGGER.info("what %s", self._ipcluster["pids"])
                    for engine_id, pid in self._ipcluster["pids"].items():
                        LOGGER.info("eid %s", engine_id)
                        LOGGER.info("pid %s", pid)
                        LOGGER.info("queue %s", ipyclient.queue_status()[engine_id]["queue"])
                        if ipyclient.queue_status()[engine_id]["queue"]:
                            LOGGER.info('interrupting engine {} w/ SIGINT to {}'\
                                        .format(engine_id, pid))
                            os.kill(pid, 2)
                    time.sleep(1)

                    ## if CLI, stop jobs and shutdown
                    if 'ipyrad-cli' in self._ipcluster["cluster_id"]:
                        LOGGER.info("  shutting down engines")
                        ipyclient.shutdown(hub=True, block=False)
                        ipyclient.close()
                        LOGGER.info("  finished shutdown")
                    else:
                        if not ipyclient.outstanding:
                            ipyclient.purge_everything()
                        else:
                            ## nanny: kill everything, something bad happened
                            ipyclient.shutdown(hub=True, block=False)
                            ipyclient.close()
                            print("\nwarning: ipcluster shutdown and must be restarted")

                ## reraise the error now that we're cleaned up
                if inst:
                    raise inst
            ## if exception is close and save, print and ignore
            except Exception as inst2:
                print("warning: error during shutdown:\n{}".format(inst2))
                LOGGER.error("shutdown warning: %s", inst2)


Instruction:
How would you explain what the following Python 3 function does
def run(data, samples, force, ipyclient):

    ## prepare dirs
    data.dirs.outfiles = os.path.join(data.dirs.project, data.name+"_outfiles")
    if not os.path.exists(data.dirs.outfiles):
        os.mkdir(data.dirs.outfiles)

    ## make the snps/filters data base, fills the dups and inds filters
    ## and fills the splits locations
    data.database = os.path.join(data.dirs.outfiles, data.name+".hdf5")
    init_arrays(data)

    ## Apply filters to supercatg and superhdf5 with selected samples
    ## and fill the filters and edge arrays.
    filter_all_clusters(data, samples, ipyclient)

    ## Everything needed is in the now filled h5 database. Filters were applied
    ## with 'samples' taken into account. Now we create the loci file (default)
    ## output and build a stats file.
    data.outfiles.loci = os.path.join(data.dirs.outfiles, data.name+".loci")
    data.outfiles.alleles = os.path.join(data.dirs.outfiles, data.name+".alleles.loci")
    make_loci_and_stats(data, samples, ipyclient)

    ## OPTIONAL OUTPUTS:
    output_formats = data.paramsdict["output_formats"]

    ## held separate from *output_formats cuz it's big and parallelized
    if any([x in output_formats for x in ["v", "V"]]):
        full = "V" in output_formats
        try:
            make_vcf(data, samples, ipyclient, full=full)
        except IPyradWarningExit as inst:
            ## Something fsck vcf build. Sometimes this is simply a memory
            ## issue, so trap the exception and allow it to try building
            ## the other output formats.
            print("  Error building vcf. See ipyrad_log.txt for details.")
            LOGGER.error(inst)

    ## make other array-based formats, recalcs keeps and arrays
    make_outfiles(data, samples, output_formats, ipyclient)

    ## print friendly message
    shortpath = data.dirs.outfiles.replace(os.path.expanduser("~"), "~")
    print("{}Outfiles written to: {}\n".format(data._spacer, shortpath))

Response: Run the clustering process.


Instruction:
Here you have a function in Python 3, explain what it does
def make_stats(data, samples, samplecounts, locuscounts):

    ## get meta info
    with h5py.File(data.clust_database, 'r') as io5:
        anames = io5["seqs"].attrs["samples"]
        nloci = io5["seqs"].shape[0]
        optim = io5["seqs"].attrs["chunksize"][0]

    ## open the out handle. This will have three data frames saved to it.
    ## locus_filtering, sample_coverages, and snp_distributions
    data.stats_files.s7 = os.path.join(data.dirs.outfiles,
                                       data.name+"_stats.txt")
    outstats = io.open(data.stats_files.s7, 'w', encoding="utf-8")

    ########################################################################
    ## get stats for locus_filtering, use chunking.
    filters = np.zeros(6, dtype=int)
    passed = 0
    start = 0

    piscounts = Counter()
    varcounts = Counter()
    for i in range(200):
        piscounts[i] = 0
        varcounts[i] = 0

    applied = pd.Series([0]*8,
        name="applied_order",
        index=[
        "total_prefiltered_loci",
        "filtered_by_rm_duplicates",
        "filtered_by_max_indels",
        "filtered_by_max_snps",
        "filtered_by_max_shared_het",
        "filtered_by_min_sample",
        "filtered_by_max_alleles",
        "total_filtered_loci"])

    ## load the h5 database
    co5 = h5py.File(data.database, 'r')

    while start < nloci:
        hslice = [start, start+optim]
        ## load each array
        afilt = co5["filters"][hslice[0]:hslice[1], ]
        asnps = co5["snps"][hslice[0]:hslice[1], ]

        ## get subarray results from filter array
        # max_indels, max_snps, max_hets, min_samps, bad_edges, max_alleles
        filters += afilt.sum(axis=0)
        applied["filtered_by_rm_duplicates"] += afilt[:, 0].sum()
        mask = afilt[:, 0].astype(np.bool)
        applied["filtered_by_max_indels"] += afilt[~mask, 1].sum()
        mask = afilt[:, 0:2].sum(axis=1).astype(np.bool)
        applied["filtered_by_max_snps"] += afilt[~mask, 2].sum()
        mask = afilt[:, 0:3].sum(axis=1).astype(np.bool)
        applied["filtered_by_max_shared_het"] += afilt[~mask, 3].sum()
        mask = afilt[:, 0:4].sum(axis=1).astype(np.bool)
        applied["filtered_by_min_sample"] += afilt[~mask, 4].sum()
        mask = afilt[:, 0:5].sum(axis=1).astype(np.bool)
        applied["filtered_by_max_alleles"] += afilt[~mask, 5].sum()

        passed += np.sum(afilt.sum(axis=1) == 0)

        ## get filter to count snps for only passed loci
        ## should we filter by all vars, or just by pis? doing all var now.
        apply_filter = afilt.sum(axis=1).astype(np.bool)

        ## get snps counts
        snplocs = asnps[~apply_filter, :].sum(axis=1)
        varlocs = snplocs.sum(axis=1)
        varcounts.update(Counter(varlocs))
        #snpcounts.update(Counter(snplocs[:, 0]))
        piscounts.update(Counter(snplocs[:, 1]))

        ## increase counter to advance through h5 database
        start += optim

    ## record filtering of loci from total to final
    filtdat = pd.Series(np.concatenate([[nloci], filters, [passed]]),
        name="total_filters",
        index=[
        "total_prefiltered_loci",
        "filtered_by_rm_duplicates",
        "filtered_by_max_indels",
        "filtered_by_max_snps",
        "filtered_by_max_shared_het",
        "filtered_by_min_sample",
        "filtered_by_max_alleles",
        "total_filtered_loci"])

    retained = pd.Series([0]*8,
        name="retained_loci",
        index=[
        "total_prefiltered_loci",
        "filtered_by_rm_duplicates",
        "filtered_by_max_indels",
        "filtered_by_max_snps",
        "filtered_by_max_shared_het",
        "filtered_by_min_sample",
        "filtered_by_max_alleles",
        "total_filtered_loci"])
    retained["total_prefiltered_loci"] = nloci
    retained["filtered_by_rm_duplicates"] = nloci - applied["filtered_by_rm_duplicates"]
    retained["filtered_by_max_indels"] = retained["filtered_by_rm_duplicates"] - applied["filtered_by_max_indels"]
    retained["filtered_by_max_snps"] = retained["filtered_by_max_indels"] - applied["filtered_by_max_snps"]
    retained["filtered_by_max_shared_het"] = retained["filtered_by_max_snps"] - applied["filtered_by_max_shared_het"]
    retained["filtered_by_min_sample"] = retained["filtered_by_max_shared_het"] - applied["filtered_by_min_sample"]
    retained["filtered_by_max_alleles"] = retained["filtered_by_min_sample"] - applied["filtered_by_max_alleles"]
    retained["total_filtered_loci"] = passed

    print(u"\n\n## The number of loci caught by each filter."+\
          u"\n## ipyrad API location: [assembly].stats_dfs.s7_filters\n",
          file=outstats)
    data.stats_dfs.s7_filters = pd.DataFrame([filtdat, applied, retained]).T
    data.stats_dfs.s7_filters.to_string(buf=outstats)

    ########################################################################
    ## make dataframe of sample_coverages
    ## samplecounts is len of anames from db. Save only samples in samples.
    #print(samplecounts)
    #samples = [i.name for i in samples]
    ## get sample names in the order of anames
    #sids = [list(anames).index(i) for i in samples]
    #covdict = {name: val for name, val in zip(np.array(samples)[sidx], samplecounts)}
    #covdict = {name: val for name, val in zip(samples, samplecounts[sidx])}
    covdict = pd.Series(samplecounts, name="sample_coverage", index=anames)
    covdict = covdict[covdict != 0]
    print(u"\n\n\n## The number of loci recovered for each Sample."+\
          u"\n## ipyrad API location: [assembly].stats_dfs.s7_samples\n",
          file=outstats)
    data.stats_dfs.s7_samples = pd.DataFrame(covdict)
    data.stats_dfs.s7_samples.to_string(buf=outstats)

    ########################################################################
    ## get stats for locus coverage
    lrange = range(1, len(samples)+1)
    locdat = pd.Series(locuscounts, name="locus_coverage", index=lrange)
    start = data.paramsdict["min_samples_locus"]-1
    locsums = pd.Series({i: np.sum(locdat.values[start:i]) for i in lrange},
                        name="sum_coverage", index=lrange)
    print(u"\n\n\n## The number of loci for which N taxa have data."+\
          u"\n## ipyrad API location: [assembly].stats_dfs.s7_loci\n",
          file=outstats)
    data.stats_dfs.s7_loci = pd.concat([locdat, locsums], axis=1)
    data.stats_dfs.s7_loci.to_string(buf=outstats)

    #########################################################################
    ## get stats for SNP_distribution
    try:
        smax = max([i+1 for i in varcounts if varcounts[i]])
    except Exception as inst:
        raise IPyradWarningExit("""
    Exception: empty varcounts array. This could be because no samples
    passed filtering, or it could be because you have overzealous filtering.
    Check the values for `trim_loci` and make sure you are not trimming the
    edge too far
    """)

    vardat = pd.Series(varcounts, name="var", index=range(smax)).fillna(0)
    sumd = {}
    for i in range(smax):
        sumd[i] = np.sum([i*vardat.values[i] for i in range(i+1)])
    varsums = pd.Series(sumd, name="sum_var", index=range(smax))

    pisdat = pd.Series(piscounts, name="pis", index=range(smax)).fillna(0)
    sumd = {}
    for i in range(smax):
        sumd[i] = np.sum([i*pisdat.values[i] for i in range(i+1)])
    pissums = pd.Series(sumd, name="sum_pis", index=range(smax))

    print(u"\n\n\n## The distribution of SNPs (var and pis) per locus."+\
          u"\n## var = Number of loci with n variable sites (pis + autapomorphies)"+\
          u"\n## pis = Number of loci with n parsimony informative site (minor allele in >1 sample)"+\
          u"\n## ipyrad API location: [assembly].stats_dfs.s7_snps\n",
          file=outstats)
    data.stats_dfs.s7_snps = pd.concat([vardat, varsums, pisdat, pissums],
                                        axis=1)
    data.stats_dfs.s7_snps.to_string(buf=outstats)

    ##########################################################################
    ## print the stats summary (-r summary) with final sample loci data.
    fullstat = data.stats
    fullstat['state'] = 7
    fullstat["loci_in_assembly"] = data.stats_dfs.s7_samples

    print(u"\n\n\n## Final Sample stats summary\n", file=outstats)
    fullstat.to_string(buf=outstats)

    ## close it
    outstats.close()
    co5.close()

Response: make stats file and save to Assembly obj


Instruction:
Can you generate the documentation for the following Python 3 function
def select_samples(dbsamples, samples, pidx=None):
    ## get index from dbsamples
    samples = [i.name for i in samples]
    if pidx:
        sidx = [list(dbsamples[pidx]).index(i) for i in samples]
    else:
        sidx = [list(dbsamples).index(i) for i in samples]
    sidx.sort()
    return sidx

Response:
Select the samples that are included in the
   .


Instruction:
Can you write a function in Python 3 where it
filters all clusters in the clust_database file with the given samples.

Response:
def filter_all_clusters(data, samples, ipyclient):
    """
    Open the clust_database HDF5 array with seqs, catg, and filter data.
    Fill the remaining filters.
    """
    ## create loadbalanced ipyclient
    lbview = ipyclient.load_balanced_view()

    ## get chunk size from the HD5 array and close
    with h5py.File(data.clust_database, 'r') as io5:
        ## the size of chunks for reading/writing
        optim = io5["seqs"].attrs["chunksize"][0]
        ## the samples in the database in their locus order
        dbsamples = io5["seqs"].attrs["samples"]
        ## the total number of loci
        nloci = io5["seqs"].shape[0]

    ## make a tmp directory for saving chunked arrays to
    chunkdir = os.path.join(data.dirs.outfiles, data.name+"_tmpchunks")
    if not os.path.exists(chunkdir):
        os.mkdir(chunkdir)

    ## get the indices of the samples that we are going to include
    sidx = select_samples(dbsamples, samples)
    ## do the same for the populations samples
    if data.populations:
        data._populations = {}
        for pop in data.populations:
            try:
                _samps = [data.samples[i] for i in data.populations[pop][1]]
                data._populations[pop] = (
                    data.populations[pop][0],
                    select_samples(dbsamples, _samps, sidx))
            except:
                print("    Sample in populations file not present in assembly - {}".format(data.populations[pop][1]))
                raise

    LOGGER.info("samples %s \n, dbsamples %s \n, sidx %s \n",
                samples, dbsamples, sidx)

    ## Put inside a try statement so we can delete tmpchunks
    try:
        ## load a list of args to send to Engines. Each arg contains the index
        ## to sample optim loci from catg, seqs, filters &or edges, which will
        ## be loaded on the remote Engine.

        ## create job queue
        start = time.time()
        printstr = " filtering loci        | {} | s7 |"
        fasyncs = {}
        submitted = 0
        while submitted < nloci:
            hslice = np.array([submitted, submitted+optim])
            fasyncs[hslice[0]] = lbview.apply(filter_stacks, *(data, sidx, hslice))
            submitted += optim

        ## run filter_stacks on all chunks
        while 1:
            readies = [i.ready() for i in fasyncs.values()]
            elapsed = datetime.timedelta(seconds=int(time.time()-start))
            progressbar(len(readies), sum(readies),
                printstr.format(elapsed), spacer=data._spacer)
            time.sleep(0.1)
            if sum(readies) == len(readies):
                print("")
                break

        ## raise error if any jobs failed
        for async in fasyncs:
            if not fasyncs[async].successful():
                LOGGER.error("error in filter_stacks on chunk %s: %s",
                             async, fasyncs[async].exception())
                raise IPyradWarningExit("error in filter_stacks on chunk {}: {}"\
                             .format(async, fasyncs[async].exception()))
        ipyclient.purge_everything()

        ## get all the saved tmp arrays for each slice
        tmpsnp = glob.glob(os.path.join(chunkdir, "snpf.*.npy"))
        tmphet = glob.glob(os.path.join(chunkdir, "hetf.*.npy"))
        tmpmin = glob.glob(os.path.join(chunkdir, "minf.*.npy"))
        tmpedg = glob.glob(os.path.join(chunkdir, "edgf.*.npy"))
        tmppld = glob.glob(os.path.join(chunkdir, "pldf.*.npy"))
        tmpind = glob.glob(os.path.join(chunkdir, "indf.*.npy"))

        ## sort array files within each group
        arrdict = OrderedDict([('ind', tmpind),
                               ('snp', tmpsnp), ('het', tmphet),
                               ('min', tmpmin), ('edg', tmpedg),
                               ('pld', tmppld)])
        for arrglob in arrdict.values():
            arrglob.sort(key=lambda x: int(x.rsplit(".")[-2]))

        ## re-load the full filter array who's order is
        ## ["duplicates", "max_indels", "max_snps", "max_hets", "min_samps", "max_alleles"]
        io5 = h5py.File(data.database, 'r+')
        superfilter = np.zeros(io5["filters"].shape, io5["filters"].dtype)

        ## iterate across filter types (dups is already filled)
        ## we have [4,4] b/c minf and edgf both write to minf
        for fidx, ftype in zip([1, 2, 3, 4, 4, 5], arrdict.keys()):
            ## fill in the edgefilters
            for ffile in arrdict[ftype]:
                ## grab a file and get it's slice
                hslice = int(ffile.split(".")[-2])
                ## load in the array
                arr = np.load(ffile)
                ## store slice into full array (we use += here because the minf
                ## and edgf arrays both write to the same filter).
                superfilter[hslice:hslice+optim, fidx] += arr

        ## store to DB
        io5["filters"][:] += superfilter
        del arr, superfilter

        ## store the other arrayed values (edges, snps)
        edgarrs = glob.glob(os.path.join(chunkdir, "edgearr.*.npy"))
        snparrs = glob.glob(os.path.join(chunkdir, "snpsarr.*.npy"))

        ## sort array files within each group
        arrdict = OrderedDict([('edges', edgarrs), ('snps', snparrs)])
        for arrglob in arrdict.values():
            arrglob.sort(key=lambda x: int(x.rsplit(".")[-2]))

        ## fill the edge array, splits are already in there.
        superedge = np.zeros(io5['edges'].shape, io5['edges'].dtype)
        for ffile in arrdict['edges']:
            ## grab a file and get it's slice
            hslice = int(ffile.split(".")[-2])
            ## load in the array w/ shape (hslice, 5)
            arr = np.load(ffile)
            ## store slice into full array
            superedge[hslice:hslice+optim, :] = arr
        io5["edges"][:, :] = superedge
        del arr, superedge

        ## fill the snps array. shape= (nloci, maxlen, 2)
        supersnps = np.zeros(io5['snps'].shape, io5['snps'].dtype)
        for ffile in arrdict['snps']:
            ## grab a file and get it's slice
            hslice = int(ffile.split(".")[-2])
            ## load in the array w/ shape (hslice, maxlen, 2)
            arr = np.load(ffile)
            ## store slice into full array
            LOGGER.info("shapes, %s %s", supersnps.shape, arr.shape)
            supersnps[hslice:hslice+optim, :, :] = arr
        io5["snps"][:] = supersnps
        del arr
        io5.close()

    finally:
        ## clean up the tmp files/dirs even if we failed.
        try:
            LOGGER.info("finished filtering")
            shutil.rmtree(chunkdir)
        except (IOError, OSError):
            pass


Instruction:
Can you write a function in Python 3 where it
pads names for loci output

Response:
def padnames(names):
    """ pads names for loci output """

    ## get longest name
    longname_len = max(len(i) for i in names)
    ## Padding distance between name and seq.
    padding = 5
    ## add pad to names
    pnames = [name + " " * (longname_len - len(name)+ padding) \
              for name in names]
    snppad = "//" + " " * (longname_len - 2 + padding)
    return np.array(pnames), snppad


Instruction:
Can you write a function in Python 3 where it
makes the. loci file from h5py data base and write stats to file.

Response:
def make_loci_and_stats(data, samples, ipyclient):
    """
    Makes the .loci file from h5 data base. Iterates by optim loci at a
    time and write to file. Also makes alleles file if requested.
    """
    ## start vcf progress bar
    start = time.time()
    printstr = " building loci/stats   | {} | s7 |"
    elapsed = datetime.timedelta(seconds=int(time.time()-start))
    progressbar(20, 0, printstr.format(elapsed), spacer=data._spacer)

    ## get some db info
    with h5py.File(data.clust_database, 'r') as io5:
        ## will iterate optim loci at a time
        optim = io5["seqs"].attrs["chunksize"][0]
        nloci = io5["seqs"].shape[0]
        anames = io5["seqs"].attrs["samples"]

    ## get name and snp padding
    pnames, snppad = padnames(anames)
    snames = [i.name for i in samples]
    smask = np.array([i not in snames for i in anames])

    ## keep track of how many loci from each sample pass all filters
    samplecov = np.zeros(len(anames), dtype=np.int32)

    ## set initial value to zero for all values above min_samples_locus
    #for cov in range(data.paramsdict["min_samples_locus"], len(anames)+1):
    locuscov = Counter()
    for cov in range(len(anames)+1):
        locuscov[cov] = 0

    ## client for sending jobs to parallel engines
    lbview = ipyclient.load_balanced_view()

    ## send jobs in chunks
    loci_asyncs = {}
    for istart in xrange(0, nloci, optim):
        args = [data, optim, pnames, snppad, smask, istart, samplecov, locuscov, 1]
        loci_asyncs[istart] = lbview.apply(locichunk, args)

    while 1:
        done = [i.ready() for i in loci_asyncs.values()]
        elapsed = datetime.timedelta(seconds=int(time.time()-start))
        progressbar(len(done), sum(done), printstr.format(elapsed), spacer=data._spacer)
        time.sleep(0.1)
        if len(done) == sum(done):
            print("")
            break

    ## check for errors
    for job in loci_asyncs:
        if loci_asyncs[job].ready() and not loci_asyncs[job].successful():
            LOGGER.error("error in building loci [%s]: %s",
                         job, loci_asyncs[job].exception())
            raise IPyradWarningExit(loci_asyncs[job].exception())

    ## concat and cleanup
    results = [i.get() for i in loci_asyncs.values()]
    ## update dictionaries
    for chunk in results:
        samplecov += chunk[0]
        locuscov.update(chunk[1])

    ## get all chunk files
    tmploci = glob.glob(data.outfiles.loci+".[0-9]*")
    ## sort by start value
    tmploci.sort(key=lambda x: int(x.split(".")[-1]))

    ## write tmpchunks to locus file
    locifile = open(data.outfiles.loci, 'w')
    for tmploc in tmploci:
        with open(tmploc, 'r') as inloc:
            locdat = inloc.read()
            locifile.write(locdat)
            os.remove(tmploc)
    locifile.close()

    ## make stats file from data
    make_stats(data, samples, samplecov, locuscov)

    ## repeat for alleles output
    if "a" in data.paramsdict["output_formats"]:

        loci_asyncs = {}
        for istart in xrange(0, nloci, optim):
            args = [data, optim, pnames, snppad, smask, istart, samplecov, locuscov, 0]
            loci_asyncs[istart] = lbview.apply(locichunk, args)

        while 1:
            done = [i.ready() for i in loci_asyncs.values()]
            elapsed = datetime.timedelta(seconds=int(time.time()-start))
            progressbar(len(done), sum(done),
                " building alleles      | {} | s7 |".format(elapsed),
                spacer=data._spacer)
            time.sleep(0.1)
            if len(done) == sum(done):
                print("")
                break

        ## check for errors
        for job in loci_asyncs:
            if loci_asyncs[job].ready() and not loci_asyncs[job].successful():
                LOGGER.error("error in building alleles [%s]: %s",
                             job, loci_asyncs[job].exception())
                raise IPyradWarningExit(loci_asyncs[job].exception())

        ## concat and cleanup
        #results = [i.get() for i in loci_asyncs.values()]

        ## get all chunk files
        tmploci = glob.glob(data.outfiles.loci+".[0-9]*")
        ## sort by start value
        tmploci.sort(key=lambda x: int(x.split(".")[-1]))

        ## write tmpchunks to locus file
        locifile = open(data.outfiles.alleles, 'w')
        for tmploc in tmploci:
            with open(tmploc, 'r') as inloc:
                locdat = inloc.read()
                inalleles = get_alleles(locdat)
                locifile.write(inalleles)
                os.remove(tmploc)
        locifile.close()


Instruction:
Can you generate a brief explanation for the following Python 3 code
def locichunk(args):
    ## parse args
    data, optim, pnames, snppad, smask, start, samplecov, locuscov, upper = args

    ## this slice
    hslice = [start, start+optim]

    ## get filter db info
    co5 = h5py.File(data.database, 'r')
    afilt = co5["filters"][hslice[0]:hslice[1], ]
    aedge = co5["edges"][hslice[0]:hslice[1], ]
    asnps = co5["snps"][hslice[0]:hslice[1], ]

    ## get seqs db
    io5 = h5py.File(data.clust_database, 'r')
    if upper:
        aseqs = np.char.upper(io5["seqs"][hslice[0]:hslice[1], ])
    else:
        aseqs = io5["seqs"][hslice[0]:hslice[1], ]

    ## which loci passed all filters
    keep = np.where(np.sum(afilt, axis=1) == 0)[0]
    store = []

    ## write loci that passed after trimming edges, then write snp string
    for iloc in keep:
        edg = aedge[iloc]
        #LOGGER.info("!!!!!! iloc edg %s, %s", iloc, edg)
        args = [iloc, pnames, snppad, edg, aseqs, asnps, smask, samplecov, locuscov, start]
        if edg[4]:
            outstr, samplecov, locuscov = enter_pairs(*args)
            store.append(outstr)
        else:
            outstr, samplecov, locuscov = enter_singles(*args)
            store.append(outstr)

    ## write to file and clear store
    tmpo = os.path.join(data.dirs.outfiles, data.name+".loci.{}".format(start))
    with open(tmpo, 'w') as tmpout:
        tmpout.write("\n".join(store) + "\n")

    ## close handles
    io5.close()
    co5.close()

    ## return sample counter
    return samplecov, locuscov, start

Response: Function from make_loci to apply to chunks. smask is sample mask. start is sample start. samplecov is sample coverage. locuscov is sample coverage. upper is True if we want to use upper = True if we want to use upper = False


Instruction:
Here you have a function in Python 3, explain what it does
def enter_pairs(iloc, pnames, snppad, edg, aseqs, asnps, smask, samplecov, locuscov, start):

    ## snps was created using only the selected samples.
    LOGGER.info("edges in enter_pairs %s", edg)
    seq1 = aseqs[iloc, :, edg[0]:edg[1]+1]
    snp1 = asnps[iloc, edg[0]:edg[1]+1, ]

    ## the 2nd read edges are +5 for the spacer
    seq2 = aseqs[iloc, :, edg[2]:edg[3]+1]
    snp2 = asnps[iloc, edg[2]:edg[3]+1, ]

    ## remove rows with all Ns, seq has only selected samples
    nalln = np.all(seq1 == "N", axis=1)

    ## make mask of removed rows and excluded samples. Use the inverse
    ## of this to save the coverage for samples
    nsidx = nalln + smask
    LOGGER.info("nsidx %s, nalln %s, smask %s", nsidx, nalln, smask)
    samplecov = samplecov + np.invert(nsidx).astype(np.int32)
    LOGGER.info("samplecov %s", samplecov)
    idx = np.sum(np.invert(nsidx).astype(np.int32))
    LOGGER.info("idx %s", idx)
    locuscov[idx] += 1

    ## select the remaining names in order
    seq1 = seq1[~nsidx, ]
    seq2 = seq2[~nsidx, ]
    names = pnames[~nsidx]

    ## save string for printing, excluding names not in samples
    outstr = "\n".join(\
        [name + s1.tostring()+"nnnn"+s2.tostring() for name, s1, s2 in \
         zip(names, seq1, seq2)])

    #LOGGER.info("s1 %s", s1.tostring())
    #LOGGER.info("s2 %s", s2.tostring())

    ## get snp string and add to store
    snpstring1 = ["-" if snp1[i, 0] else \
                 "*" if snp1[i, 1] else \
                 " " for i in range(len(snp1))]
    snpstring2 = ["-" if snp2[i, 0] else \
                 "*" if snp2[i, 1] else \
                 " " for i in range(len(snp2))]

    #npis = str(snpstring1+snpstring2).count("*")
    #nvars = str(snpstring1+snpstring2).count("-") + npis
    outstr += "\n" + snppad + "".join(snpstring1)+\
              "    "+"".join(snpstring2)+"|{}|".format(iloc+start)
              #"|LOCID={},DBID={},NVAR={},NPIS={}|"\
              #.format(1+iloc+start, iloc, nvars, npis)

    return outstr, samplecov, locuscov

Response: enter_pairs is the function that enters the pairs of the input sequences