Instruction:
Write a Python 3 function that can
take a string argument and returns value as a tuple.

Response:
def _tuplecheck(newvalue, dtype=str):
    """
    Takes a string argument and returns value as a tuple.
    Needed for paramfile conversion from CLI to set_params args
    """

    if isinstance(newvalue, list):
        newvalue = tuple(newvalue)

    if isinstance(newvalue, str):
        newvalue = newvalue.rstrip(")").strip("(")
        try:
            newvalue = tuple([dtype(i.strip()) for i in newvalue.split(",")])

        ## Type error is thrown by tuple if it's applied to a non-iterable.
        except TypeError:
            newvalue = tuple(dtype(newvalue))

        ## If dtype fails to cast any element of newvalue
        except ValueError:
            LOGGER.info("Assembly.tuplecheck() failed to cast to {} - {}"\
                        .format(dtype, newvalue))
            raise

        except Exception as inst:
            LOGGER.info(inst)
            raise SystemExit(\
            "\nError: Param`{}` is not formatted correctly.\n({})\n"\
                 .format(newvalue, inst))

    return newvalue


Instruction:
Write a Python 3 function that can
check if the given parameter is set to values.

Response:
def _paramschecker(self, param, newvalue):
    """ Raises exceptions when params are set to values they should not be"""

    if param == 'assembly_name':
        ## Make sure somebody doesn't try to change their assembly_name, bad
        ## things would happen. Calling set_params on assembly_name only raises
        ## an informative error. Assembly_name is set at Assembly creation time
        ## and is immutable.
        raise IPyradWarningExit(CANNOT_CHANGE_ASSEMBLY_NAME)

    elif param == 'project_dir':
        expandpath = _expander(newvalue)
        if not expandpath.startswith("/"):
            if os.path.exists(expandpath):
                expandpath = _expander(expandpath)
        ## Forbid spaces in path names
        if " " in expandpath:
            raise IPyradWarningExit(BAD_PROJDIR_NAME.format(expandpath))
        self.paramsdict["project_dir"] = expandpath
        self.dirs["project"] = expandpath

    ## `Merged:` in newvalue for raw_fastq_path indicates that this
    ## assembly is a merge of several others, so this param has no
    ## value for this assembly
    elif param == 'raw_fastq_path':
        if newvalue and not "Merged:" in newvalue:
            fullrawpath = _expander(newvalue)
            if os.path.isdir(fullrawpath):
                raise IPyradWarningExit(RAW_PATH_ISDIR.format(fullrawpath))
            ## if something is found in path
            elif glob.glob(fullrawpath):
                self.paramsdict['raw_fastq_path'] = fullrawpath
            ## else allow empty, tho it can still raise an error in step1
            else:
                raise IPyradWarningExit(NO_RAW_FILE.format(fullrawpath))
        else:
            self.paramsdict['raw_fastq_path'] = ""

    ## `Merged:` in newvalue for barcodes_path indicates that this
    ## assembly is a merge of several others, so this param has no
    ## value for this assembly
    elif param == 'barcodes_path':
        ## if a value was entered check that it exists
        if newvalue and not "Merged:" in newvalue:
            ## also allow for fuzzy match in names using glob
            fullbarpath = glob.glob(_expander(newvalue))[0]
            ## raise error if file is not found
            if not os.path.exists(fullbarpath):
                raise IPyradWarningExit(BARCODE_NOT_FOUND.format(fullbarpath))
            else:
                self.paramsdict['barcodes_path'] = fullbarpath
                self._link_barcodes()

        ## if no path was entered then set barcodes path to empty.
        ## this is checked again during step 1 and will raise an error
        ## if you try demultiplexing without a barcodes file
        else:
            self.paramsdict['barcodes_path'] = newvalue

    ## `Merged:` in newvalue for sorted_fastq_path indicates that this
    ## assembly is a merge of several others, so this param has no
    ## value for this assembly
    elif param == 'sorted_fastq_path':
        if newvalue and not "Merged:" in newvalue:
            fullsortedpath = _expander(newvalue)

            if os.path.isdir(fullsortedpath):
                raise IPyradWarningExit(SORTED_ISDIR.format(fullsortedpath))
            elif glob.glob(fullsortedpath):
                self.paramsdict['sorted_fastq_path'] = fullsortedpath

            else:
                raise IPyradWarningExit(SORTED_NOT_FOUND.format(fullsortedpath))
        ## if no value was entered then set to "".
        else:
            self.paramsdict['sorted_fastq_path'] = ""

    elif param == 'assembly_method':
        ## TEMPORARY BLOCK ON DENOVO+REFERENCE METHOD
#        if newvalue == "denovo+reference":
#            raise IPyradWarningExit("""
#    Error: The 'denovo+reference' method is temporarily blocked while we
#    refactor it to greatly improve the speed. You can either revert to an
#    older version (pre v.0.7.0) or wait for the next update to resume using
#    this method.
#    """)

        methods = ["denovo", "reference", "denovo+reference", "denovo-reference"]
        assert newvalue in methods, BAD_ASSEMBLY_METHOD.format(newvalue)
        self.paramsdict['assembly_method'] = newvalue

    elif param == 'reference_sequence':
        if newvalue:
            fullrawpath = _expander(newvalue)
            if not os.path.isfile(fullrawpath):
                LOGGER.info("reference sequence file not found.")
                raise IPyradWarningExit(REF_NOT_FOUND.format(fullrawpath))
            self.paramsdict['reference_sequence'] = fullrawpath
        ## if no value was entered the set to "". Will be checked again
        ## at step3 if user tries to do refseq and raise error
        else:
            self.paramsdict['reference_sequence'] = ""

    elif param == 'datatype':
        ## list of allowed datatypes
        datatypes = ['rad', 'gbs', 'ddrad', 'pairddrad',
                     'pairgbs', 'merged', '2brad', 'pair3rad']
        ## raise error if something else
        if str(newvalue) not in datatypes:
            raise IPyradError("""
    datatype {} not recognized, must be one of: {}
    """.format(newvalue, datatypes))
        else:
            self.paramsdict['datatype'] = str(newvalue)
            ## link_barcodes is called before datatypes is set
            ## we need to know the datatype so we can read in
            ## the multiplexed barcodes for 3rad. This seems
            ## a little annoying, but it was better than any
            ## alternatives I could think of.
            if "3rad" in self.paramsdict['datatype'] and not \
            self.paramsdict['sorted_fastq_path'].strip():
                if not "Merged:" in self.paramsdict['barcodes_path']:
                    self._link_barcodes()

    elif param == 'restriction_overhang':
        newvalue = _tuplecheck(newvalue, str)
        assert isinstance(newvalue, tuple), """
    cut site must be a tuple, e.g., (TGCAG, '') or (TGCAG, CCGG)"""
        ## Handle the special case where the user has 1
        ## restriction overhang and does not include the trailing comma
        if len(newvalue) == 1:
            ## for gbs users might not know to enter the second cut site
            ## so we do it for them.
            if self.paramsdict["datatype"] == "gbs":
                newvalue += newvalue
            else:
                newvalue += ("",)
        #=======
        #    newvalue = (newvalue[0], "")
        #>>>>>>> d40a5d5086a0d0aace04dd08338ec4ba5341d1f2

        ## Handle 3rad datatype with only 3 cutters
        if len(newvalue) == 3:
            newvalue = (newvalue[0], newvalue[1], newvalue[2], "")
        assert len(newvalue) <= 4, """
    most datasets require 1 or 2 cut sites, e.g., (TGCAG, '') or (TGCAG, CCGG).
    For 3rad/seqcap may be up to 4 cut sites."""
        self.paramsdict['restriction_overhang'] = newvalue

    elif param == 'max_low_qual_bases':
        assert isinstance(int(newvalue), int), """
    max_low_qual_bases must be an integer."""
        self.paramsdict['max_low_qual_bases'] = int(newvalue)

    elif param == 'phred_Qscore_offset':
        assert isinstance(int(newvalue), int), \
            "phred_Qscore_offset must be an integer."
        self.paramsdict['phred_Qscore_offset'] = int(newvalue)

    elif param == 'mindepth_statistical':
        assert isinstance(int(newvalue), int), \
            "mindepth_statistical must be an integer."
        ## do not allow values below 5
        if int(newvalue) < 5:
            raise IPyradError("""
    mindepth statistical cannot be set < 5. Use mindepth_majrule.""")
        else:
            self.paramsdict['mindepth_statistical'] = int(newvalue)

    elif param == 'mindepth_majrule':
        assert isinstance(int(newvalue), int), \
            "mindepth_majrule must be an integer."
        self.paramsdict['mindepth_majrule'] = int(newvalue)

    elif param == 'maxdepth':
        self.paramsdict['maxdepth'] = int(newvalue)

    elif param == 'clust_threshold':
        newvalue = float(newvalue)
        assert (newvalue < 1) & (newvalue > 0), \
        "clust_threshold must be a decimal value between 0 and 1."
        self.paramsdict['clust_threshold'] = newvalue

    elif param == 'max_barcode_mismatch':
        self.paramsdict['max_barcode_mismatch'] = int(newvalue)

    elif param == 'filter_adapters':
        self.paramsdict['filter_adapters'] = int(newvalue)

    elif param == 'filter_min_trim_len':
        self.paramsdict["filter_min_trim_len"] = int(newvalue)

    elif param == 'max_alleles_consens':
        self.paramsdict['max_alleles_consens'] = int(newvalue)

    elif param == 'max_Ns_consens':
        newvalue = _tuplecheck(newvalue, int)
        assert isinstance(newvalue, tuple), \
        "max_Ns_consens should be a tuple e.g., (8, 8)"
        self.paramsdict['max_Ns_consens'] = newvalue

    elif param == 'max_Hs_consens':
        newvalue = _tuplecheck(newvalue, int)
        assert isinstance(newvalue, tuple), \
        "max_Hs_consens should be a tuple e.g., (5, 5)"
        self.paramsdict['max_Hs_consens'] = newvalue

    elif param == 'min_samples_locus':
        self.paramsdict['min_samples_locus'] = int(newvalue)

    elif param == 'max_shared_Hs_locus':
        if isinstance(newvalue, str):
            if newvalue.isdigit():
                newvalue = int(newvalue)
            else:
                try:
                    newvalue = float(newvalue)
                except Exception as inst:
                    raise IPyradParamsError("""
    max_shared_Hs_locus must be int or float, you put: {}""".format(newvalue))
        self.paramsdict['max_shared_Hs_locus'] = newvalue

    elif param == 'max_SNPs_locus':
        newvalue = _tuplecheck(newvalue, int)
        assert isinstance(newvalue, tuple), \
        "max_SNPs_locus should be a tuple e.g., (20, 20)"
        self.paramsdict['max_SNPs_locus'] = newvalue

    elif param == 'max_Indels_locus':
        newvalue = _tuplecheck(newvalue, int)
        assert isinstance(newvalue, tuple), \
        "max_Indels_locus should be a tuple e.g., (5, 100)"
        self.paramsdict['max_Indels_locus'] = newvalue

    ## deprecated but retained for legacy, now uses trim_reads (below)
    elif param == 'edit_cutsites':
        ## Force into a string tuple
        newvalue = _tuplecheck(newvalue)
        ## try converting each tup element to ints
        newvalue = list(newvalue)
        for i in range(2):
            try:
                newvalue[i] = int(newvalue[i])
            except (ValueError, IndexError):
                newvalue.append(0)
                pass
        newvalue = tuple(newvalue)
        ## make sure we have a nice tuple
        if not isinstance(newvalue, tuple):
            raise IPyradWarningExit("""
    Error: edit_cutsites should be a tuple e.g., (0, 5) or ('TGCAG', 6),
    you entered {}
    """.format(newvalue))
        self.paramsdict['edit_cutsites'] = newvalue

    elif param == 'trim_reads':
        ## Force into a string tuple
        newvalue = _tuplecheck(newvalue)
        ## try converting each tup element to ints
        newvalue = list(newvalue)
        for i in range(4):
            try:
                newvalue[i] = int(newvalue[i])
            except (ValueError, IndexError):
                newvalue.append(0)
                pass
        newvalue = tuple(newvalue)
        ## make sure we have a nice tuple
        if not isinstance(newvalue, tuple):
            raise IPyradWarningExit("""
    Error: trim_reads should be a tuple e.g., (0, -5, -5, 0)
    or (0, 90, 0, 90), or (0, 0, 0, 0).
    You entered {}\n""".format(newvalue))
        self.paramsdict['trim_reads'] = newvalue

    ## deprecated but retained for legacy, now named trim_loci
    elif param == 'trim_overhang':
        newvalue = _tuplecheck(newvalue, str)
        assert isinstance(newvalue, tuple), \
        "trim_overhang should be a tuple e.g., (4, *, *, 4)"
        self.paramsdict['trim_overhang'] = tuple([int(i) for i in newvalue])

    elif param == 'trim_loci':
        newvalue = _tuplecheck(newvalue, str)
        assert isinstance(newvalue, tuple), \
        "trim_overhang should be a tuple e.g., (0, -5, -5, 0)"
        self.paramsdict['trim_loci'] = tuple([int(i) for i in newvalue])

    elif param == 'output_formats':
        ## let's get whatever the user entered as a tuple of letters
        allowed = assemble.write_outfiles.OUTPUT_FORMATS.keys()

        #<<<<<<< HEAD
        ## Handle the case where output formats is an empty string
        if isinstance(newvalue, str):
            ## strip commas and spaces from string so we have only letters
            newvalue = newvalue.replace(",", "").replace(" ", "")
            newvalue = list(newvalue)
            if not newvalue:
                newvalue = ["*"]
        if isinstance(newvalue, tuple):
            newvalue = list(newvalue)
        #=======
        #if isinstance(newvalue, tuple):
        #    newvalue = list(newvalue)
        #if isinstance(newvalue, str):
        #    newvalue = [i.strip() for i in newvalue.split(",")]
        #    ## Handle the case where output formats is empty
        #    if not any(newvalue):
        #        newvalue = "*"
        #>>>>>>> 488144d1d97240b8b6f6caf9cfb6c023bb6ebb36
        if isinstance(newvalue, list):
            ## if more than letters, raise an warning
            if any([len(i) > 1 for i in newvalue]):
                LOGGER.warning("""
    'output_formats' params entry is malformed. Setting to * to avoid errors.""")
                newvalue = allowed
            newvalue = tuple(newvalue)
            #newvalue = tuple([i for i in newvalue if i in allowed])
        if "*" in newvalue:
            newvalue = allowed

        ## set the param
        self.paramsdict['output_formats'] = newvalue

    elif param == 'pop_assign_file':
        fullpoppath = _expander(newvalue)

        ## if a path is entered, raise exception if not found
        if newvalue:
            if not os.path.isfile(fullpoppath):
                LOGGER.warn("Population assignment file not found.")
                raise IPyradWarningExit("""
    Warning: Population assignment file not found. This must be an
    absolute path (/home/wat/ipyrad/data/my_popfile.txt) or relative to
    the directory where you're running ipyrad (./data/my_popfile.txt)
    You entered: {}\n""".format(fullpoppath))
        ## should we add a check here that all pop samples are in samples?

            self.paramsdict['pop_assign_file'] = fullpoppath
            self._link_populations()
        else:
            self.paramsdict['pop_assign_file'] = ""
            ## Don't forget to possibly blank the populations dictionary
            self.populations = {}

    return self


Instruction:
Write a Python 3 script for
returning a data frame with Sample data and state.

Response:
def stats(self):
        """ Returns a data frame with Sample data and state. """
        nameordered = self.samples.keys()
        nameordered.sort()

        ## Set pandas to display all samples instead of truncating
        pd.options.display.max_rows = len(self.samples)
        statdat = pd.DataFrame([self.samples[i].stats for i in nameordered],
                      index=nameordered).dropna(axis=1, how='all')
        # ensure non h,e columns print as ints
        for column in statdat:
            if column not in ["hetero_est", "error_est"]:
                statdat[column] = np.nan_to_num(statdat[column]).astype(int)
        return statdat


Instruction:
Make a summary of the following Python 3 code
def files(self):
        nameordered = self.samples.keys()
        nameordered.sort()
        ## replace curdir with . for shorter printing
        #fullcurdir = os.path.realpath(os.path.curdir)
        return pd.DataFrame([self.samples[i].files for i in nameordered],
                      index=nameordered).dropna(axis=1, how='all')

Response: Returns a data frame with Sample files.


Instruction:
Make a summary of the following Python 3 code
def _build_stat(self, idx):
        nameordered = self.samples.keys()
        nameordered.sort()
        newdat = pd.DataFrame([self.samples[i].stats_dfs[idx] \
                               for i in nameordered], index=nameordered)\
                               .dropna(axis=1, how='all')
        return newdat

Response: Builds a data frame with Sample stats for each step


Instruction:
Implement a Python 3 function for
linking all fastq files in the specified path to the specified path.

Response:
def _link_fastqs(self, path=None, force=False, append=False, splitnames="_",
        fields=None, ipyclient=None):
        """
        Create Sample objects from demultiplexed fastq files in sorted_fastq_path,
        or append additional fastq files to existing Samples. This provides
        more flexible file input through the API than available in step1 of the
        command line interface. If passed ipyclient it will run in parallel.

        Note
        ----
        This function is called during step 1 if files are specified in
        'sorted_fastq_path'.

        Parameters
        ----------
        path : str
            Path to the fastq files to be linked to Sample objects. The default
            location is to select all files in the 'sorted_fastq_path'.
            Alternatively a different path can be entered here.

        append : bool
            The default action is to overwrite fastq files linked to Samples if
            they already have linked files. Use append=True to instead append
            additional fastq files to a Sample (file names should be formatted
            the same as usual, e.g., [name]_R1_[optional].fastq.gz).

        splitnames : str
            A string character used to file names. In combination with the
            fields argument can be used to subselect filename fields names.

        fields : list
            A list of indices for the fields to be included in names after
            filnames are split on the splitnames character. Useful for appending
            sequence names which must match existing names. If the largest index
            is greater than the number of split strings in the name the index
            if ignored. e.g., [2,3,4] ## excludes 0, 1 and >4

        force : bool
            Overwrites existing Sample data and statistics.

        Returns
        -------
        str
            Prints the number of new Sample objects created and the number of
            fastq files linked to Sample objects in the Assembly object.
        """

        ## cannot both force and append at once
        if force and append:
            raise IPyradError("Cannot use force and append at the same time.")

        if self.samples and not (force or append):
            raise IPyradError("Files already linked to `{}`.".format(self.name)\
                +" Use force=True to replace all files, or append=True to add"
                +" additional files to existing Samples.")

        ## make sure there is a workdir and workdir/fastqdir
        self.dirs.fastqs = os.path.join(self.paramsdict["project_dir"],
                                        self.name+"_fastqs")
        if not os.path.exists(self.paramsdict["project_dir"]):
            os.mkdir(self.paramsdict["project_dir"])

        ## get path to data files
        if not path:
            path = self.paramsdict["sorted_fastq_path"]

        ## but grab fastq/fq/gz, and then sort
        fastqs = glob.glob(path)
        ## Assert files are not .bz2 format
        if any([i for i in fastqs if i.endswith(".bz2")]):
            raise IPyradError(NO_SUPPORT_FOR_BZ2.format(path))
        fastqs = [i for i in fastqs if i.endswith(".gz") \
                                    or i.endswith(".fastq") \
                                    or i.endswith(".fq")]
        fastqs.sort()
        LOGGER.debug("Linking these fastq files:\n{}".format(fastqs))

        ## raise error if no files are found
        if not fastqs:
            raise IPyradError(NO_FILES_FOUND_PAIRS\
                        .format(self.paramsdict["sorted_fastq_path"]))

        ## link pairs into tuples
        if 'pair' in self.paramsdict["datatype"]:
            ## check that names fit the paired naming convention
            ## trying to support flexible types (_R2_, _2.fastq)
            r1_try1 = [i for i in fastqs if "_R1_" in i]
            r1_try2 = [i for i in fastqs if i.endswith("_1.fastq.gz")]
            r1_try3 = [i for i in fastqs if i.endswith("_R1.fastq.gz")]

            r2_try1 = [i for i in fastqs if "_R2_" in i]
            r2_try2 = [i for i in fastqs if i.endswith("_2.fastq.gz")]
            r2_try3 = [i for i in fastqs if i.endswith("_R2.fastq.gz")]

            r1s = [r1_try1, r1_try2, r1_try3]
            r2s = [r2_try1, r2_try2, r2_try3]

            ## check that something was found
            if not r1_try1 + r1_try2 + r1_try3:
                raise IPyradWarningExit(
                    "Paired filenames are improperly formatted. See Documentation")
            if not r2_try1 + r2_try2 + r2_try3:
                raise IPyradWarningExit(
                    "Paired filenames are improperly formatted. See Documentation")

            ## find the one with the right number of R1s
            for idx, tri in enumerate(r1s):
                if len(tri) == len(fastqs)/2:
                    break
            r1_files = r1s[idx]
            r2_files = r2s[idx]

            if len(r1_files) != len(r2_files):
                raise IPyradWarningExit(R1_R2_name_error\
                    .format(len(r1_files), len(r2_files)))

            fastqs = [(i, j) for i, j in zip(r1_files, r2_files)]

        ## data are not paired, create empty tuple pair
        else:
            ## print warning if _R2_ is in names when not paired
            idx = 0
            if any(["_R2_" in i for i in fastqs]):
                print(NAMES_LOOK_PAIRED_WARNING)
            fastqs = [(i, "") for i in fastqs]

        ## counters for the printed output
        linked = 0
        appended = 0

        ## clear samples if force
        if force:
            self.samples = {}

        ## track parallel jobs
        linkjobs = {}
        if ipyclient:
            lbview = ipyclient.load_balanced_view()

        ## iterate over input files
        for fastqtuple in list(fastqs):
            assert isinstance(fastqtuple, tuple), "fastqs not a tuple."
            ## local counters
            createdinc = 0
            linkedinc = 0
            appendinc = 0
            ## remove file extension from name
            if idx == 0:
                sname = _name_from_file(fastqtuple[0], splitnames, fields)
            elif idx == 1:
                sname = os.path.basename(fastqtuple[0].rsplit("_1.fastq.gz", 1)[0])
            elif idx == 2:
                sname = os.path.basename(fastqtuple[0].rsplit("_R1.fastq.gz", 1)[0])
            LOGGER.debug("New Sample name {}".format(sname))

            if sname not in self.samples:
                ## create new Sample
                LOGGER.debug("Creating new sample - ".format(sname))
                self.samples[sname] = Sample(sname)
                self.samples[sname].stats.state = 1
                self.samples[sname].barcode = None
                self.samples[sname].files.fastqs.append(fastqtuple)
                createdinc += 1
                linkedinc += 1
            else:
                ## if not forcing, shouldn't be here with existing Samples
                if append:
                    #if fastqtuple not in self.samples[sname].files.fastqs:
                    self.samples[sname].files.fastqs.append(fastqtuple)
                    appendinc += 1

                elif force:
                    ## overwrite/create new Sample
                    LOGGER.debug("Overwriting sample - ".format(sname))
                    self.samples[sname] = Sample(sname)
                    self.samples[sname].stats.state = 1
                    self.samples[sname].barcode = None
                    self.samples[sname].files.fastqs.append(fastqtuple)
                    createdinc += 1
                    linkedinc += 1
                else:
                    print("""
        The files {} are already in Sample. Use append=True to append additional
        files to a Sample or force=True to replace all existing Samples.
        """.format(sname))

            ## support serial execution w/o ipyclient
            if not ipyclient:
                if any([linkedinc, createdinc, appendinc]):
                    gzipped = bool(fastqtuple[0].endswith(".gz"))
                    nreads = 0
                    for alltuples in self.samples[sname].files.fastqs:
                        nreads += _zbufcountlines(alltuples[0], gzipped)
                    self.samples[sname].stats.reads_raw = nreads/4
                    self.samples[sname].stats_dfs.s1["reads_raw"] = nreads/4
                    self.samples[sname].state = 1

                    LOGGER.debug("Got reads for sample - {} {}".format(sname,\
                                    self.samples[sname].stats.reads_raw))
                    #created += createdinc
                    linked += linkedinc
                    appended += appendinc

            ## do counting in parallel
            else:
                if any([linkedinc, createdinc, appendinc]):
                    gzipped = bool(fastqtuple[0].endswith(".gz"))
                    for sidx, tup in enumerate(self.samples[sname].files.fastqs):
                        key = sname+"_{}".format(sidx)
                        linkjobs[key] = lbview.apply(_bufcountlines,
                                                    *(tup[0], gzipped))
                    LOGGER.debug("sent count job for {}".format(sname))
                    #created += createdinc
                    linked += linkedinc
                    appended += appendinc

        ## wait for link jobs to finish if parallel
        if ipyclient:
            start = time.time()
            printstr = ' loading reads         | {} | s1 |'
            while 1:
                fin = [i.ready() for i in linkjobs.values()]
                elapsed = datetime.timedelta(seconds=int(time.time()-start))
                progressbar(len(fin), sum(fin),
                    printstr.format(elapsed), spacer=self._spacer)
                time.sleep(0.1)
                if len(fin) == sum(fin):
                    print("")
                    break

            ## collect link job results
            sampdict = {i:0 for i in self.samples}
            for result in linkjobs:
                sname = result.rsplit("_", 1)[0]
                nreads = linkjobs[result].result()
                sampdict[sname] += nreads

            for sname in sampdict:
                self.samples[sname].stats.reads_raw = sampdict[sname]/4
                self.samples[sname].stats_dfs.s1["reads_raw"] = sampdict[sname]/4
                self.samples[sname].state = 1

        ## print if data were linked
        #print("  {} new Samples created in '{}'.".format(created, self.name))
        if linked:
            ## double for paired data
            if 'pair' in self.paramsdict["datatype"]:
                linked = linked*2
            if self._headers:
                print("{}{} fastq files loaded to {} Samples.".\
                      format(self._spacer, linked, len(self.samples)))
            ## save the location where these files are located
            self.dirs.fastqs = os.path.realpath(os.path.dirname(path))

        if appended:
            if self._headers:
                print("{}{} fastq files appended to {} existing Samples.".\
                      format(self._spacer, appended, len(self.samples)))

        ## save step-1 stats. We don't want to write this to the fastq dir, b/c
        ## it is not necessarily inside our project dir. Instead, we'll write
        ## this file into our project dir in the case of linked_fastqs.
        self.stats_dfs.s1 = self._build_stat("s1")
        self.stats_files.s1 = os.path.join(self.paramsdict["project_dir"],
                                           self.name+
                                           '_s1_demultiplex_stats.txt')
        with open(self.stats_files.s1, 'w') as outfile:
            self.stats_dfs.s1.fillna(value=0).astype(np.int).to_string(outfile)


Instruction:
How would you explain what the following Python 3 function does
def _link_barcodes(self):

        ## parse barcodefile
        try:
            ## allows fuzzy match to barcodefile name
            barcodefile = glob.glob(self.paramsdict["barcodes_path"])[0]

            ## read in the file
            bdf = pd.read_csv(barcodefile, header=None, delim_whitespace=1, dtype=str)
            bdf = bdf.dropna()

            ## make sure bars are upper case
            bdf[1] = bdf[1].str.upper()

            ## if replicates are present then print a warning
            reps = bdf[0].unique().shape[0] != bdf[0].shape[0]
            if reps:
                print("{spacer}Warning: technical replicates (same name) will be combined."\
                      .format(**{'spacer': self._spacer}))
                ## add -technical-replicate-N to replicate names
                reps = [i for i in bdf[0] if list(bdf[0]).count(i) > 1]
                ureps = list(set(reps))
                for name in ureps:
                    idxs = bdf[bdf[0] == ureps[0]].index.tolist()
                    for num, idx in enumerate(idxs):
                        bdf.ix[idx][0] = bdf.ix[idx][0] + "-technical-replicate-" + str(num+1)

            ## make sure chars are all proper
            if not all(bdf[1].apply(set("RKSYWMCATG").issuperset)):
                LOGGER.warn(BAD_BARCODE)
                raise IPyradError(BAD_BARCODE)

            ## 3rad/seqcap use multiplexed barcodes
            ## We'll concatenate them with a plus and split them later
            if "3rad" in self.paramsdict["datatype"]:
                try:
                    bdf[2] = bdf[2].str.upper()
                    self.barcodes = dict(zip(bdf[0], bdf[1] + "+" + bdf[2]))
                except KeyError as inst:
                    msg = "    3rad assumes multiplexed barcodes. Doublecheck your barcodes file."
                    LOGGER.error(msg)
                    raise IPyradError(msg)
            else:
                ## set attribute on Assembly object
                self.barcodes = dict(zip(bdf[0], bdf[1]))

        except (IOError, IndexError):
            raise IPyradWarningExit(\
                "    Barcodes file not found. You entered: {}"\
                .format(self.paramsdict["barcodes_path"]))

        except ValueError as inst:
            msg = "    Barcodes file format error."
            LOGGER.warn(msg)
            raise IPyradError(inst)

Response:
Private function that links sample barcodes in a dictionary as
           .


Instruction:
Write a Python 3 function that can
create a dictionary that contains the populations for a specific loci and the number of samples that are present in each locus.

Response:
def _link_populations(self, popdict=None, popmins=None):
        """
        Creates self.populations dictionary to save mappings of individuals to
        populations/sites, and checks that individual names match with Samples.
        The self.populations dict keys are pop names and the values are lists
        of length 2. The first element is the min number of samples per pop
        for final filtering of loci, and the second element is the list of
        samples per pop.

        Population assigments are used for heirarchical clustering, for
        generating summary stats, and for outputing some file types (.treemix
        for example). Internally stored as a dictionary.

        Note
        ----
        By default a File is read in from `pop_assign_file` with one individual
        per line and space separated pairs of ind pop:

            ind1 pop1
            ind2 pop2
            ind3 pop3
            etc...

        Parameters
        ----------
        TODO: NB: Using API and passing in popdict and popmins is currently
        unimplemented, or at least looks like it doesn't work. Leaving
        these docs cuz Deren might have ideas about it being useful.

        popdict : dict
            When using the API it may be easier to simply create a dictionary
            to pass in as an argument instead of reading from an input file.
            This can be done with the `popdict` argument like below:

            pops = {'pop1': ['ind1', 'ind2', 'ind3'], 'pop2': ['ind4', 'ind5']}
            [Assembly]._link_populations(popdict=pops).

        popmins : dict
            If you want to apply a minsamples filter based on populations
            you can add a popmins dictionary. This indicates the number of
            samples in each population that must be present in a locus for
            the locus to be retained. Example:

            popmins = {'pop1': 3, 'pop2': 2}

        """
        if not popdict:
            ## glob it in case of fuzzy matching
            popfile = glob.glob(self.paramsdict["pop_assign_file"])[0]
            if not os.path.exists(popfile):
                raise IPyradError("Population assignment file not found: {}"\
                                  .format(self.paramsdict["pop_assign_file"]))

            try:
                ## parse populations file
                popdat = pd.read_csv(popfile, header=None,
                                              delim_whitespace=1,
                                              names=["inds", "pops"],
                                              comment="#")
                popdict = {key: group.inds.values.tolist() for key, group in \
                                                        popdat.groupby("pops")}

                ## parse minsamples per population if present (line with #)
                mindat = [i.lstrip("#").lstrip().rstrip() for i in \
                          open(popfile, 'r').readlines() if i.startswith("#")]
                if mindat:
                    popmins = {}
                    for i in range(len(mindat)):
                        minlist = mindat[i].replace(",", "").split()
                        popmins.update({i.split(':')[0]:int(i.split(':')[1]) \
                                        for i in minlist})
                else:
                    raise IPyradError(MIN_SAMPLES_PER_POP_MALFORMED)

            except (ValueError, IOError):
                LOGGER.warn("Populations file may be malformed.")
                raise IPyradError(MIN_SAMPLES_PER_POP_MALFORMED)

        else:
            ## pop dict is provided by user
            pass

        ## check popdict. Filter for bad samples
        ## Warn user but don't bail out, could be setting the pops file
        ## on a new assembly w/o any linked samples.
        badsamples = [i for i in itertools.chain(*popdict.values()) \
                      if i not in self.samples.keys()]
        if any(badsamples):
            LOGGER.warn("Some names from population input do not match Sample "\
                        + "names: ".format(", ".join(badsamples)))
            LOGGER.warn("If this is a new assembly this is normal.")

        ## If popmins not set, just assume all mins are zero
        if not popmins:
            popmins = {i: 0 for i in popdict.keys()}

        ## check popmins
        ## cannot have higher min for a pop than there are samples in the pop
        popmax = {i: len(popdict[i]) for i in popdict}
        if not all([popmax[i] >= popmins[i] for i in popdict]):
            raise IPyradWarningExit(\
                " minsample per pop value cannot be greater than the "+
                " number of samples in the pop. Modify the populations file.")

        ## return dict
        self.populations = {i: (popmins[i], popdict[i]) for i in popdict}


Instruction:
Explain what the following Python 3 code does
def set_params(self, param, newvalue):

        ## this includes current params and some legacy params for conversion
        legacy_params = ["edit_cutsites", "trim_overhang"]
        current_params = self.paramsdict.keys()
        allowed_params = current_params + legacy_params

        ## require parameter recognition
        #if not ((param in range(50)) or \
        #        (param in [str(i) for i in range(50)]) or \
        #        (param in allowed_params)):
        if not param in allowed_params:
            raise IPyradParamsError("Parameter key not recognized: {}"\
                                    .format(param))

        ## make string
        param = str(param)
        ## get index if param is keyword arg (this index is now zero based!)
        if len(param) < 3:
            param = self.paramsdict.keys()[int(param)]

        ## run assertions on new param
        try:
            self = _paramschecker(self, param, newvalue)

        except Exception as inst:
            raise IPyradWarningExit(BAD_PARAMETER\
                                    .format(param, inst, newvalue))

Response: Set a parameter in the current Assembly object to a new value. Raises error if newvalue is wrong type.


Instruction:
Explain what the following Python 3 code does
def write_params(self, outfile=None, force=False):
        if outfile is None:
            outfile = "params-"+self.name+".txt"

        ## Test if params file already exists?
        ## If not forcing, test for file and bail out if it exists
        if not force:
            if os.path.isfile(outfile):
                raise IPyradWarningExit(PARAMS_EXISTS.format(outfile))

        with open(outfile, 'w') as paramsfile:
            ## Write the header. Format to 80 columns
            header = "------- ipyrad params file (v.{})".format(ip.__version__)
            header += ("-"*(80-len(header)))
            paramsfile.write(header)

            ## Whip through the current paramsdict and write out the current
            ## param value, the ordered dict index number. Also,
            ## get the short description from paramsinfo. Make it look pretty,
            ## pad nicely if at all possible.
            for key, val in self.paramsdict.iteritems():
                ## If multiple elements, write them out comma separated
                if isinstance(val, list) or isinstance(val, tuple):
                    paramvalue = ", ".join([str(i) for i in val])
                else:
                    paramvalue = str(val)

                ## skip deprecated params
                if key in ["edit_cutsites", "trim_overhang"]:
                    continue

                padding = (" "*(30-len(paramvalue)))
                paramkey = self.paramsdict.keys().index(key)
                paramindex = " ## [{}] ".format(paramkey)
                LOGGER.debug(key, val, paramindex)
                name = "[{}]: ".format(paramname(paramkey))
                description = paraminfo(paramkey, short=True)
                paramsfile.write("\n" + paramvalue + padding + \
                                        paramindex + name + description)

Response: Write out the parameters of this assembly to a file properly with the parameters of the current assembly.


Instruction:
Explain what the following Python 3 code does
def branch(self, newname, subsamples=None, infile=None):
        ## subsample by removal or keeping.
        remove = 0

        ## is there a better way to ask if it already exists?
        if (newname == self.name or os.path.exists(
                                    os.path.join(self.paramsdict["project_dir"],
                                    newname+".assembly"))):
            print("{}Assembly object named {} already exists"\
                  .format(self._spacer, newname))

        else:
            ## Make sure the new name doesn't have any wacky characters
            self._check_name(newname)

            ## Bozo-check. Carve off 'params-' if it's in the new name.
            if newname.startswith("params-"):
                newname = newname.split("params-")[1]

            ## create a copy of the Assembly obj
            newobj = copy.deepcopy(self)
            newobj.name = newname
            newobj.paramsdict["assembly_name"] = newname

            if subsamples and infile:
                print(BRANCH_NAMES_AND_INPUT)

            if infile:
                if infile[0] == "-":
                    remove = 1
                    infile = infile[1:]
                if os.path.exists(infile):
                    subsamples = _read_sample_names(infile)

            ## if remove then swap the samples
            if remove:
                subsamples = list(set(self.samples.keys()) - set(subsamples))

            ## create copies of each subsampled Sample obj
            if subsamples:
                for sname in subsamples:
                    if sname in self.samples:
                        newobj.samples[sname] = copy.deepcopy(self.samples[sname])
                    else:
                        print("Sample name not found: {}".format(sname))

                ## reload sample dict w/o non subsamples
                newobj.samples = {name:sample for name, sample in \
                           newobj.samples.items() if name in subsamples}

            ## create copies of each subsampled Sample obj
            else:
                for sample in self.samples:
                    newobj.samples[sample] = copy.deepcopy(self.samples[sample])

            ## save json of new obj and return object
            newobj.save()
            return newobj

Response: Branch the Assembly object with the new name.


Instruction:
Create a Python 3 function for
hiding wrapped function to start the demultiplexing step 1

Response:
def _step1func(self, force, ipyclient):
        """ hidden wrapped function to start step 1 """

        ## check input data files
        sfiles = self.paramsdict["sorted_fastq_path"]
        rfiles = self.paramsdict["raw_fastq_path"]

        ## do not allow both a sorted_fastq_path and a raw_fastq
        if sfiles and rfiles:
            raise IPyradWarningExit(NOT_TWO_PATHS)

        ## but also require that at least one exists
        if not (sfiles or rfiles):
            raise IPyradWarningExit(NO_SEQ_PATH_FOUND)

        ## print headers
        if self._headers:
            if sfiles:
                print("\n{}Step 1: Loading sorted fastq data to Samples"\
                      .format(self._spacer))
            else:
                print("\n{}Step 1: Demultiplexing fastq data to Samples"\
                    .format(self._spacer))

        ## if Samples already exist then no demultiplexing
        if self.samples:
            if not force:
                print(SAMPLES_EXIST.format(len(self.samples), self.name))
            else:
                ## overwrite existing data else do demux
                if glob.glob(sfiles):
                    self._link_fastqs(ipyclient=ipyclient, force=force)
                else:
                    assemble.demultiplex.run2(self, ipyclient, force)

        ## Creating new Samples
        else:
            ## first check if demultiplexed files exist in sorted path
            if glob.glob(sfiles):
                self._link_fastqs(ipyclient=ipyclient)

            ## otherwise do the demultiplexing
            else:
                assemble.demultiplex.run2(self, ipyclient, force)


Instruction:
Create a Python 3 function to
hide wrapped function to start step 2

Response:
def _step2func(self, samples, force, ipyclient):
        """ hidden wrapped function to start step 2"""

        ## print header
        if self._headers:
            print("\n  Step 2: Filtering reads ")

        ## If no samples in this assembly then it means you skipped step1,
        if not self.samples.keys():
            raise IPyradWarningExit(FIRST_RUN_1)

        ## Get sample objects from list of strings, if API.
        samples = _get_samples(self, samples)

        if not force:
            ## print warning and skip if all are finished
            if all([i.stats.state >= 2 for i in samples]):
                print(EDITS_EXIST.format(len(samples)))
                return

        ## Run samples through rawedit
        assemble.rawedit.run2(self, samples, force, ipyclient)


Instruction:
Write a Python 3 script for
running the clustering within step 3

Response:
def _step3func(self, samples, noreverse, maxindels, force, ipyclient):
        """ hidden wrapped function to start step 3 """
        ## print headers
        if self._headers:
            print("\n  Step 3: Clustering/Mapping reads")

        ## Require reference seq for reference-based methods
        if self.paramsdict['assembly_method'] != "denovo":
            if not self.paramsdict['reference_sequence']:
                raise IPyradError(REQUIRE_REFERENCE_PATH\
                            .format(self.paramsdict["assembly_method"]))
            else:
                ## index the reference sequence
                ## Allow force to reindex the reference sequence
                ## send to run on the cluster.
                lbview = ipyclient.load_balanced_view()
                async = lbview.apply(index_reference_sequence, *(self, force))

                ## print a progress bar for the indexing
                start = time.time()
                while 1:
                    elapsed = datetime.timedelta(seconds=int(time.time()-start))
                    printstr = " {}    | {} | s3 |".format("indexing reference", elapsed)
                    finished = int(async.ready())
                    progressbar(1, finished, printstr, spacer=self._spacer)
                    if finished:
                        print("")
                        break
                    time.sleep(0.9)
                ## error check
                if not async.successful():
                    raise IPyradWarningExit(async.result())

        ## Get sample objects from list of strings
        samples = _get_samples(self, samples)

        ## Check if all/none in the right state
        if not self._samples_precheck(samples, 3, force):
            raise IPyradError(FIRST_RUN_2)

        elif not force:
            ## skip if all are finished
            if all([i.stats.state >= 3 for i in samples]):
                print(CLUSTERS_EXIST.format(len(samples)))
                return

        ## run the step function
        assemble.cluster_within.run(self, samples, noreverse, maxindels,
                                    force, ipyclient)


Instruction:
How would you implement a function in Python 3 that
hides wrapped function to start step 4

Response:
def _step4func(self, samples, force, ipyclient):
        """ hidden wrapped function to start step 4 """

        if self._headers:
            print("\n  Step 4: Joint estimation of error rate and heterozygosity")

        ## Get sample objects from list of strings
        samples = _get_samples(self, samples)

        ## Check if all/none in the right state
        if not self._samples_precheck(samples, 4, force):
            raise IPyradError(FIRST_RUN_3)

        elif not force:
            ## skip if all are finished
            if all([i.stats.state >= 4 for i in samples]):
                print(JOINTS_EXIST.format(len(samples)))
                return

        ## send to function
        assemble.jointestimate.run(self, samples, force, ipyclient)


Instruction:
Write a Python 3 script to
hide wrapped function to start the step 5

Response:
def _step5func(self, samples, force, ipyclient):
        """ hidden wrapped function to start step 5 """
        ## print header
        if self._headers:
            print("\n  Step 5: Consensus base calling ")

        ## Get sample objects from list of strings
        samples = _get_samples(self, samples)

        ## Check if all/none in the right state
        if not self._samples_precheck(samples, 5, force):
            raise IPyradError(FIRST_RUN_4)

        elif not force:
            ## skip if all are finished
            if all([i.stats.state >= 5 for i in samples]):
                print(CONSENS_EXIST.format(len(samples)))
                return
        ## pass samples to rawedit
        assemble.consens_se.run(self, samples, force, ipyclient)


Instruction:
Write a Python 3 script to
hide function to start Step 6.

Response:
def _step6func(self,
        samples,
        noreverse,
        force,
        randomseed,
        ipyclient,
        **kwargs):
        """
        Hidden function to start Step 6.
        """

        ## Get sample objects from list of strings
        samples = _get_samples(self, samples)

        ## remove samples that aren't ready
        csamples = self._samples_precheck(samples, 6, force)

        ## print CLI header
        if self._headers:
            print("\n  Step 6: Clustering at {} similarity across {} samples".\
                  format(self.paramsdict["clust_threshold"], len(csamples)))

        ## Check if all/none in the right state
        if not csamples:
            raise IPyradError(FIRST_RUN_5)

        elif not force:
            ## skip if all are finished
            if all([i.stats.state >= 6 for i in csamples]):
                print(DATABASE_EXISTS.format(len(samples)))
                return

        ## run if this point is reached. We no longer check for existing
        ## h5 file, since checking Sample states should suffice.
        assemble.cluster_across.run(
            self,
            csamples,
            noreverse,
            force,
            randomseed,
            ipyclient,
            **kwargs)


Instruction:
How would you code a function in Python 3 to
step 7: Filter and write output files

Response:
def _step7func(self, samples, force, ipyclient):
        """ Step 7: Filter and write output files """

        ## Get sample objects from list of strings
        samples = _get_samples(self, samples)

        if self._headers:
            print("\n  Step 7: Filter and write output files for {} Samples".\
                  format(len(samples)))

        ## Check if all/none of the samples are in the self.database
        try:
            with h5py.File(self.clust_database, 'r') as io5:
                dbset = set(io5["seqs"].attrs['samples'])
                iset = set([i.name for i in samples])

                ## TODO: Handle the case where dbdiff is not empty?
                ## This might arise if someone tries to branch and remove
                ## samples at step 7.
                dbdiff = dbset.difference(iset)
                idiff = iset.difference(dbset)
                if idiff:
                    print(NOT_CLUSTERED_YET\
                    .format(self.database, ", ".join(list(idiff))))

                    ## The the old way that failed unless all samples were
                    ## clustered successfully in step 6. Adding some flexibility
                    ## to allow writing output even if some samples failed.
                    ## raise IPyradWarningExit(msg)

                    ## Remove the samples that aren't ready for writing out
                    ## i.e. only proceed with the samples that are actually
                    ## present in the db
                    samples = [x for x in samples if x.name not in idiff]
        except (IOError, ValueError):
            raise IPyradError(FIRST_RUN_6.format(self.clust_database))

        if not force:
            outdir = os.path.join(self.dirs.project, self.name+"_outfiles")
            if os.path.exists(outdir):
                raise IPyradWarningExit(OUTPUT_EXISTS.format(outdir))

        ## Run step7
        assemble.write_outfiles.run(self, samples, force, ipyclient)


Instruction:
Can you implement a function in Python 3 that
returns a list of samples that are actually ready for the next step.

Response:
def _samples_precheck(self, samples, mystep, force):
        """ Return a list of samples that are actually ready for the next step.
            Each step runs this prior to calling run, makes it easier to
            centralize and normalize how each step is checking sample states.
            mystep is the state produced by the current step.
        """
        subsample = []
        ## filter by state
        for sample in samples:
            if sample.stats.state < mystep - 1:
                LOGGER.debug("Sample {} not in proper state."\
                             .format(sample.name))
            else:
                subsample.append(sample)
        return subsample


Instruction:
How would you explain what the following Python 3 function does
def _compatible_params_check(self):

        ## do not allow statistical < majrule
        val1 = self.paramsdict["mindepth_statistical"]
        val2 = self.paramsdict['mindepth_majrule']
        if val1 < val2:
            msg = """
    Warning: mindepth_statistical cannot not be < mindepth_majrule.
    Forcing mindepth_majrule = mindepth_statistical = {}
    """.format(val1)
            LOGGER.warning(msg)
            print(msg)
            self.paramsdict["mindepth_majrule"] = val1

Response: check for mindepths after all params are set


Instruction:
Here you have a function in Python 3, explain what it does
def run(self, steps=0, force=False, ipyclient=None,
        show_cluster=0, **kwargs):
        """
        Run assembly steps of an ipyrad analysis. Enter steps as a string,
        e.g., "1", "123", "12345". This step checks for an existing
        ipcluster instance otherwise it raises an exception. The ipyparallel
        connection is made using information from the _ipcluster dict of the
        Assembly class object.
        """
        ## check that mindepth params are compatible, fix and report warning.
        self._compatible_params_check()

        ## wrap everything in a try statement to ensure that we save the
        ## Assembly object if it is interrupted at any point, and also
        ## to ensure proper cleanup of the ipyclient.
        inst = None
        try:
            ## use an existing ipcluster instance
            if not ipyclient:
                args = self._ipcluster.items() + [("spacer", self._spacer)]
                ipyclient = ip.core.parallel.get_client(**dict(args))

            ## print a message about the cluster status
            ## if MPI setup then we are going to wait until all engines are
            ## ready so that we can print how many cores started on each
            ## host machine exactly.
            if (self._cli) or show_cluster:
                ip.cluster_info(ipyclient=ipyclient, spacer=self._spacer)

            ## get the list of steps to run
            if isinstance(steps, int):
                steps = str(steps)
            steps = sorted(list(steps))

            ## print an Assembly name header if inside API
            if not self._cli:
                print("Assembly: {}".format(self.name))

            ## store ipyclient engine pids to the Assembly so we can
            ## hard-interrupt them later if assembly is interrupted.
            ## Only stores pids of engines that aren't busy at this moment,
            ## otherwise it would block here while waiting to find their pids.
            self._ipcluster["pids"] = {}
            for eid in ipyclient.ids:
                engine = ipyclient[eid]
                if not engine.outstanding:
                    pid = engine.apply(os.getpid).get()
                    self._ipcluster["pids"][eid] = pid
            #ipyclient[:].apply(os.getpid).get_dict()

            ## has many fixed arguments right now, but we may add these to
            ## hackerz_only, or they may be accessed in the API.
            if '1' in steps:
                self._step1func(force, ipyclient)
                self.save()
                ipyclient.purge_everything()

            if '2' in steps:
                self._step2func(samples=None, force=force, ipyclient=ipyclient)
                self.save()
                ipyclient.purge_everything()

            if '3' in steps:
                self._step3func(samples=None, noreverse=0, force=force,
                             maxindels=8, ipyclient=ipyclient)
                self.save()
                ipyclient.purge_everything()

            if '4' in steps:
                self._step4func(samples=None, force=force, ipyclient=ipyclient)
                self.save()
                ipyclient.purge_everything()

            if '5' in steps:
                self._step5func(samples=None, force=force, ipyclient=ipyclient)
                self.save()
                ipyclient.purge_everything()

            if '6' in steps:
                self._step6func(samples=None, noreverse=0, randomseed=12345,
                                force=force, ipyclient=ipyclient, **kwargs)
                self.save()
                ipyclient.purge_everything()

            if '7' in steps:
                self._step7func(samples=None, force=force, ipyclient=ipyclient)
                self.save()
                ipyclient.purge_everything()

        ## handle exceptions so they will be raised after we clean up below
        except KeyboardInterrupt as inst:
            print("\n  Keyboard Interrupt by user")
            LOGGER.info("assembly interrupted by user.")

        except IPyradWarningExit as inst:
            LOGGER.error("IPyradWarningExit: %s", inst)
            print("\n  Encountered an error (see details in ./ipyrad_log.txt)"+\
                  "\n  Error summary is below -------------------------------"+\
                  "\n{}".format(inst))

        except Exception as inst:
            LOGGER.error(inst)
            print("\n  Encountered an unexpected error (see ./ipyrad_log.txt)"+\
                  "\n  Error message is below -------------------------------"+\
                  "\n{}".format(inst))

        ## close client when done or interrupted
        finally:
            try:
                ## save the Assembly
                self.save()

                ## can't close client if it was never open
                if ipyclient:

                    ## send SIGINT (2) to all engines
                    try:
                        ipyclient.abort()
                        time.sleep(1)
                        for engine_id, pid in self._ipcluster["pids"].items():
                            if ipyclient.queue_status()[engine_id]["tasks"]:
                                os.kill(pid, 2)
                                LOGGER.info('interrupted engine {} w/ SIGINT to {}'\
                                        .format(engine_id, pid))
                        time.sleep(1)
                    except ipp.NoEnginesRegistered:
                        pass

                    ## if CLI, stop jobs and shutdown. Don't use _cli here
                    ## because you can have a CLI object but use the --ipcluster
                    ## flag, in which case we don't want to kill ipcluster.
                    if 'ipyrad-cli' in self._ipcluster["cluster_id"]:
                        LOGGER.info("  shutting down engines")
                        ipyclient.shutdown(hub=True, block=False)
                        ipyclient.close()
                        LOGGER.info("  finished shutdown")
                    else:
                        if not ipyclient.outstanding:
                            ipyclient.purge_everything()
                        else:
                            ## nanny: kill everything, something bad happened
                            ipyclient.shutdown(hub=True, block=False)
                            ipyclient.close()
                            print("\nwarning: ipcluster shutdown and must be restarted")

            ## if exception is close and save, print and ignore
            except Exception as inst2:
                print("warning: error during shutdown:\n{}".format(inst2))
                LOGGER.error("shutdown warning: %s", inst2)

Response: Run assembly steps of an ipyrad analysis.


Instruction:
Explain what the following Python 3 code does
def _to_fulldict(self):

        ##
        returndict = OrderedDict([
            ("name", self.name),
            ("barcode", self.barcode),
            ("files", self.files),
            ("stats_dfs", {
                "s1": self.stats_dfs.s1.to_dict(),
                "s2": self.stats_dfs.s2.to_dict(),
                "s3": self.stats_dfs.s3.to_dict(),
                "s4": self.stats_dfs.s4.to_dict(),
                "s5": self.stats_dfs.s5.to_dict(),
            }),
            ("stats", self.stats.to_dict()),
            ("depths", self.depths)
            ])

        return returndict

Response:
Write to dict including data frames. All sample dicts
        are combined in dump (


Instruction:
Can you create a Python 3 function that
combines first and second read files into one sequence.

Response:
def combinefiles(filepath):
    """ Joins first and second read file names """
    ## unpack seq files in filepath
    fastqs = glob.glob(filepath)
    firsts = [i for i in fastqs if "_R1_" in i]

    ## check names
    if not firsts:
        raise IPyradWarningExit("First read files names must contain '_R1_'.")

    ## get paired reads
    seconds = [ff.replace("_R1_", "_R2_") for ff in firsts]
    return zip(firsts, seconds)


Instruction:
Create a Python 3 function to
find barcode sequence in the beginning of read

Response:
def findbcode(cutters, longbar, read1):
    """ find barcode sequence in the beginning of read """
    ## default barcode string
    for cutter in cutters[0]:
        ## If the cutter is unambiguous there will only be one.
        if not cutter:
            continue
        search = read1[1][:int(longbar[0]+len(cutter)+1)]
        barcode = search.rsplit(cutter, 1)
        if len(barcode) > 1:
            return barcode[0]
    ## No cutter found
    return barcode[0]


Instruction:
How would you explain what the following Python 3 function does
def find3radbcode(cutters, longbar, read1):
    ## default barcode string
    for ambigcuts in cutters:
        for cutter in ambigcuts:
            ## If the cutter is unambiguous there will only be one.
            if not cutter:
                continue
            search = read1[1][:int(longbar[0]+len(cutter)+1)]
            splitsearch = search.rsplit(cutter, 1)
            if len(splitsearch) > 1:
                return splitsearch[0]
    ## No cutter found
    return splitsearch[0]

Response: find 3radbcode in the beginning of a read


Instruction:
How would you explain what the following Python 3 function does
def make_stats(data, perfile, fsamplehits, fbarhits, fmisses, fdbars):

    ## out file
    outhandle = os.path.join(data.dirs.fastqs, 's1_demultiplex_stats.txt')
    outfile = open(outhandle, 'w')

    ## write the header for file stats ------------------------------------
    outfile.write('{:<35}  {:>13}{:>13}{:>13}\n'.\
                  format("raw_file", "total_reads", "cut_found", "bar_matched"))

    ## write the file stats
    r1names = sorted(perfile)
    for fname in r1names:
        dat = perfile[fname]
        #dat = [perfile[fname][i] for i in ["ftotal", "fcutfound", "fmatched"]]
        outfile.write('{:<35}  {:>13}{:>13}{:>13}\n'.\
            format(fname, dat[0], dat[1], dat[2]))
        ## repeat for pairfile
        if 'pair' in data.paramsdict["datatype"]:
            fname = fname.replace("_R1_", "_R2_")
            outfile.write('{:<35}  {:>13}{:>13}{:>13}\n'.\
                format(fname, dat[0], dat[1], dat[2]))

    ## spacer, how many records for each sample --------------------------
    outfile.write('\n{:<35}  {:>13}\n'.format("sample_name", "total_reads"))

    ## names alphabetical. Write to file. Will save again below to Samples.
    snames = set()
    for sname in data.barcodes:
        if "-technical-replicate-" in sname:
            sname = sname.rsplit("-technical-replicate", 1)[0]
        snames.add(sname)

    for sname in sorted(list(snames)):
        outfile.write("{:<35}  {:>13}\n".format(sname, fsamplehits[sname]))

    ## spacer, which barcodes were found -----------------------------------
    outfile.write('\n{:<35}  {:>13} {:>13} {:>13}\n'.\
                  format("sample_name", "true_bar", "obs_bar", "N_records"))

    ## write sample results
    for sname in sorted(data.barcodes):
        if "-technical-replicate-" in sname:
            fname = sname.rsplit("-technical-replicate", 1)[0]
        else:
            fname = sname

        ## write perfect hit
        hit = data.barcodes[sname]
        offhitstring = ""

        ## write off-n hits
        ## sort list of off-n hits
        if fname in fdbars:
            offkeys = list(fdbars.get(fname))
            for offhit in offkeys[::-1]:
                ## exclude perfect hit
                if offhit not in data.barcodes.values():
                    offhitstring += '{:<35}  {:>13} {:>13} {:>13}\n'.\
                        format(sname, hit, offhit, fbarhits[offhit]/2)
                    #sumoffhits += fbarhits[offhit]

            ## write string to file
            outfile.write('{:<35}  {:>13} {:>13} {:>13}\n'.\
                #format(sname, hit, hit, fsamplehits[fname]-sumoffhits))
                format(sname, hit, hit, fbarhits[hit]/2))
            outfile.write(offhitstring)

    ## write misses
    misskeys = list(fmisses.keys())
    misskeys.sort(key=fmisses.get)
    for key in misskeys[::-1]:
        outfile.write('{:<35}  {:>13}{:>13}{:>13}\n'.\
            format("no_match", "_", key, fmisses[key]))
    outfile.close()

    ## Link Sample with this data file to the Assembly object
    for sname in snames:

        ## make the sample
        sample = Sample()
        sample.name = sname

        ## allow multiple barcodes if its a replicate.
        barcodes = []
        for n in xrange(500):
            fname = sname+"-technical-replicate-{}".format(n)
            fbar = data.barcodes.get(fname)
            if fbar:
                barcodes.append(fbar)
        if barcodes:
            sample.barcode = barcodes
        else:
            sample.barcode = data.barcodes[sname]

        ## file names
        if 'pair' in data.paramsdict["datatype"]:
            sample.files.fastqs = [(os.path.join(data.dirs.fastqs,
                                                  sname+"_R1_.fastq.gz"),
                                     os.path.join(data.dirs.fastqs,
                                                  sname+"_R2_.fastq.gz"))]
        else:
            sample.files.fastqs = [(os.path.join(data.dirs.fastqs,
                                                  sname+"_R1_.fastq.gz"), "")]
        ## fill in the summary stats
        sample.stats["reads_raw"] = int(fsamplehits[sname])
        ## fill in the full df stats value
        sample.stats_dfs.s1["reads_raw"] = int(fsamplehits[sname])

        ## Only link Sample if it has data
        if sample.stats["reads_raw"]:
            sample.stats.state = 1
            data.samples[sample.name] = sample
        else:
            print("Excluded sample: no data found for", sname)

    ## initiate s1 key for data object
    data.stats_dfs.s1 = data._build_stat("s1")
    data.stats_files.s1 = outhandle

Response: Write stats and stores to Assembly object.


Instruction:
Explain what the following Python 3 code does
def get_barcode_func(data, longbar):
    ## build func for finding barcode
    if longbar[1] == 'same':
        if data.paramsdict["datatype"] == '2brad':
            def getbarcode(cutters, read1, longbar):
                """ find barcode for 2bRAD data """
                return read1[1][:-(len(cutters[0][0]) + 1)][-longbar[0]:]

        else:
            def getbarcode(_, read1, longbar):
                """ finds barcode for invariable length barcode data """
                return read1[1][:longbar[0]]
    else:
        def getbarcode(cutters, read1, longbar):
            """ finds barcode for variable barcode lengths"""
            return findbcode(cutters, longbar, read1)
    return getbarcode

Response: returns the fastest function given data & longbar


Instruction:
How would you explain what the following Python 3 function does
def get_quart_iter(tups):

    if tups[0].endswith(".gz"):
        ofunc = gzip.open
    else:
        ofunc = open

    ## create iterators
    ofile1 = ofunc(tups[0], 'r')
    fr1 = iter(ofile1)
    quart1 = itertools.izip(fr1, fr1, fr1, fr1)
    if tups[1]:
        ofile2 = ofunc(tups[1], 'r')
        fr2 = iter(ofile2)
        quart2 = itertools.izip(fr2, fr2, fr2, fr2)
        quarts = itertools.izip(quart1, quart2)
    else:
        ofile2 = 0
        quarts = itertools.izip(quart1, iter(int, 1))

    ## make a generator
    def feedme(quarts):
        for quart in quarts:
            yield quart
    genquarts = feedme(quarts)

    ## return generator and handles
    return genquarts, ofile1, ofile2

Response: returns an iterator to grab four lines at a time


Instruction:
Explain what the following Python 3 code does
def writetofastq(data, dsort, read):
    if read == 1:
        rrr = "R1"
    else:
        rrr = "R2"

    for sname in dsort:
        ## skip writing if empty. Write to tmpname
        handle = os.path.join(data.dirs.fastqs,
                "{}_{}_.fastq".format(sname, rrr))
        with open(handle, 'a') as out:
            out.write("".join(dsort[sname]))

Response:
Writes sorted data to a tmp files
AttributeNames


Instruction:
Can you create a Python 3 function that
collates temp fastq files in tmp - dir into 1 gzipped sample.

Response:
def collate_files(data, sname, tmp1s, tmp2s):
    """
    Collate temp fastq files in tmp-dir into 1 gzipped sample.
    """
    ## out handle
    out1 = os.path.join(data.dirs.fastqs, "{}_R1_.fastq.gz".format(sname))
    out = io.BufferedWriter(gzip.open(out1, 'w'))

    ## build cmd
    cmd1 = ['cat']
    for tmpfile in tmp1s:
        cmd1 += [tmpfile]

    ## compression function
    proc = sps.Popen(['which', 'pigz'], stderr=sps.PIPE, stdout=sps.PIPE).communicate()
    if proc[0].strip():
        compress = ["pigz"]
    else:
        compress = ["gzip"]

    ## call cmd
    proc1 = sps.Popen(cmd1, stderr=sps.PIPE, stdout=sps.PIPE)
    proc2 = sps.Popen(compress, stdin=proc1.stdout, stderr=sps.PIPE, stdout=out)
    err = proc2.communicate()
    if proc2.returncode:
        raise IPyradWarningExit("error in collate_files R1 %s", err)
    proc1.stdout.close()
    out.close()

    ## then cleanup
    for tmpfile in tmp1s:
        os.remove(tmpfile)

    if 'pair' in data.paramsdict["datatype"]:
        ## out handle
        out2 = os.path.join(data.dirs.fastqs, "{}_R2_.fastq.gz".format(sname))
        out = io.BufferedWriter(gzip.open(out2, 'w'))

        ## build cmd
        cmd1 = ['cat']
        for tmpfile in tmp2s:
            cmd1 += [tmpfile]

        ## call cmd
        proc1 = sps.Popen(cmd1, stderr=sps.PIPE, stdout=sps.PIPE)
        proc2 = sps.Popen(compress, stdin=proc1.stdout, stderr=sps.PIPE, stdout=out)
        err = proc2.communicate()
        if proc2.returncode:
            raise IPyradWarningExit("error in collate_files R2 %s", err)
        proc1.stdout.close()
        out.close()

        ## then cleanup
        for tmpfile in tmp2s:
            os.remove(tmpfile)


Instruction:
Given the following Python 3 function, write the documentation
def prechecks2(data, force):

    ## check for data using glob for fuzzy matching
    if not glob.glob(data.paramsdict["raw_fastq_path"]):
        raise IPyradWarningExit(NO_RAWS.format(data.paramsdict["raw_fastq_path"]))

    ## find longest barcode
    try:
        ## Handle 3rad multi-barcodes. Gets len of the first one.
        ## Should be harmless for single barcode data
        barlens = [len(i.split("+")[0]) for i in data.barcodes.values()]
        if len(set(barlens)) == 1:
            longbar = (barlens[0], 'same')
        else:
            longbar = (max(barlens), 'diff')

        ## For 3rad we need to add the length info for barcodes_R2
        if "3rad" in data.paramsdict["datatype"]:
            barlens = [len(i.split("+")[1]) for i in data.barcodes.values()]
            longbar = (longbar[0], longbar[1], max(barlens))
    except ValueError:
        raise IPyradWarningExit(NO_BARS.format(data.paramsdict["barcodes_path"]))

    ## setup dirs: [workdir] and a [workdir/name_fastqs]
    opj = os.path.join

    ## create project dir
    pdir = os.path.realpath(data.paramsdict["project_dir"])
    if not os.path.exists(pdir):
        os.mkdir(pdir)

    ## create fastq dir
    data.dirs.fastqs = opj(pdir, data.name+"_fastqs")
    if os.path.exists(data.dirs.fastqs) and force:
        print(OVERWRITING_FASTQS.format(**{"spacer":data._spacer}))
        shutil.rmtree(data.dirs.fastqs)
    if not os.path.exists(data.dirs.fastqs):
        os.mkdir(data.dirs.fastqs)

    ## insure no leftover tmp files from a previous run (there shouldn't be)
    oldtmps = glob.glob(os.path.join(data.dirs.fastqs, "tmp_*_R1_"))
    oldtmps += glob.glob(os.path.join(data.dirs.fastqs, "tmp_*_R2_"))
    for oldtmp in oldtmps:
        os.remove(oldtmp)

    ## gather raw sequence filenames (people want this to be flexible ...)
    if 'pair' in data.paramsdict["datatype"]:
        raws = combinefiles(data.paramsdict["raw_fastq_path"])
    else:
        raws = zip(glob.glob(data.paramsdict["raw_fastq_path"]), iter(int, 1))

    ## returns a list of both resolutions of cut site 1
    ## (TGCAG, ) ==> [TGCAG, ]
    ## (TWGC, ) ==> [TAGC, TTGC]
    ## (TWGC, AATT) ==> [TAGC, TTGC]
    cutters = [ambigcutters(i) for i in data.paramsdict["restriction_overhang"]]
    print(cutters)
    assert cutters, "Must enter a `restriction_overhang` for demultiplexing."

    ## get matchdict
    matchdict = inverse_barcodes(data)

    ## return all
    return raws, longbar, cutters, matchdict

Response:
This function is a simplified version of demux
   . It is used to demux the entire tree of files.


Instruction:
Write a Python 3 script for
building a full inverse barcodes dictionary

Response:
def inverse_barcodes(data):
    """ Build full inverse barcodes dictionary """

    matchdict = {}
    bases = set("CATGN")
    poss = set()

    ## do perfect matches
    for sname, barc in data.barcodes.items():
        ## remove -technical-replicate-N if present
        if "-technical-replicate-" in sname:
            sname = sname.rsplit("-technical-replicate", 1)[0]
        matchdict[barc] = sname
        poss.add(barc)

        if data.paramsdict["max_barcode_mismatch"] > 0:
            ## get 1-base diffs
            for idx1, base in enumerate(barc):
                diffs = bases.difference(base)
                for diff in diffs:
                    lbar = list(barc)
                    lbar[idx1] = diff
                    tbar1 = "".join(lbar)
                    if tbar1 not in poss:
                        matchdict[tbar1] = sname
                        poss.add(tbar1)
                    else:
                        if matchdict.get(tbar1) != sname:
                            print("""\
        Note: barcodes {}:{} and {}:{} are within {} base change of each other
            Ambiguous barcodes that match to both samples will arbitrarily
            be assigned to the first sample. If you do not like this idea
            then lower the value of max_barcode_mismatch and rerun step 1\n"""\
        .format(sname, barc,
                matchdict[tbar1], data.barcodes[matchdict[tbar1]],
                data.paramsdict["max_barcode_mismatch"]))

                ## if allowing two base difference things get big
                ## for each modified bar, allow one modification to other bases
                if data.paramsdict["max_barcode_mismatch"] > 1:
                    for idx2, _ in enumerate(tbar1):
                        ## skip the base that is already modified
                        if idx2 != idx1:
                            for diff in bases.difference(tbar1[idx2]):
                                ltbar = list(tbar1)
                                ltbar[idx2] = diff
                                tbar2 = "".join(ltbar)
                                if tbar2 not in poss:
                                    matchdict[tbar2] = sname
                                    poss.add(tbar2)
                                else:
                                    if matchdict.get(tbar2) != sname:
                                        print("""\
        Note: barcodes {}:{} and {}:{} are within {} base change of each other\
             Ambiguous barcodes that match to both samples will arbitrarily
             be assigned to the first sample. If you do not like this idea
             then lower the value of max_barcode_mismatch and rerun step 1\n"""\
        .format(sname, barc,
                            matchdict[tbar2], data.barcodes[matchdict[tbar2]],
                            data.paramsdict["max_barcode_mismatch"]))
    return matchdict


Instruction:
How would you explain what the following Python 3 function does
def estimate_optim(data, testfile, ipyclient):
    ## count the len of one file and assume all others are similar len
    insize = os.path.getsize(testfile)
    tmp_file_name = os.path.join(data.paramsdict["project_dir"], "tmp-step1-count.fq")
    if testfile.endswith(".gz"):
        infile = gzip.open(testfile)
        outfile = gzip.open(tmp_file_name, 'wb', compresslevel=5)
    else:
        infile = open(testfile)
        outfile = open(tmp_file_name, 'w')

    ## We'll take the average of the size of a file based on the
    ## first 10000 reads to approximate number of reads in the main file
    outfile.write("".join(itertools.islice(infile, 40000)))
    outfile.close()
    infile.close()

    ## Get the size of the tmp file
    tmp_size = os.path.getsize(tmp_file_name)

    ## divide by the tmp file size and multiply by 10000 to approximate
    ## the size of the input .fq files
    inputreads = int(insize / tmp_size) * 10000
    os.remove(tmp_file_name)

    return inputreads

Response: Estimate a reasonable optim value by grabbing a chunk of sequences decompressing and counting them to estimate the full file size.


Instruction:
Given the following Python 3 function, write the documentation
def run2(data, ipyclient, force):

    ## get file handles, name-lens, cutters, and matchdict
    raws, longbar, cutters, matchdict = prechecks2(data, force)

    ## wrap funcs to ensure we can kill tmpfiles
    kbd = 0
    try:
        ## if splitting files, split files into smaller chunks for demuxing
        chunkfiles = splitfiles(data, raws, ipyclient)

        ## send chunks to be demux'd
        statdicts = demux2(data, chunkfiles, cutters, longbar, matchdict, ipyclient)

        ## concat tmp files
        concat_chunks(data, ipyclient)

        ## build stats from dictionaries
        perfile, fsamplehits, fbarhits, fmisses, fdbars = statdicts
        make_stats(data, perfile, fsamplehits, fbarhits, fmisses, fdbars)

    except KeyboardInterrupt:
        print("\n  ...interrupted, just a second while we ensure proper cleanup")
        kbd = 1

    ## cleanup
    finally:
        ## cleaning up the tmpdir is safe from ipyclient
        tmpdir = os.path.join(data.paramsdict["project_dir"], "tmp-chunks-"+data.name)
        if os.path.exists(tmpdir):
            shutil.rmtree(tmpdir)

        if kbd:
            raise KeyboardInterrupt("s1")
        else:
            _cleanup_and_die(data)

Response: This function is a helper function that runs two processors on a single input file or pair.


Instruction:
Can you generate the documentation for the following Python 3 function
def _cleanup_and_die(data):
    tmpfiles = glob.glob(os.path.join(data.dirs.fastqs, "tmp_*_R*.fastq"))
    tmpfiles += glob.glob(os.path.join(data.dirs.fastqs, "tmp_*.p"))
    for tmpf in tmpfiles:
        os.remove(tmpf)

Response: cleanup func for step 1


Instruction:
Create a Python 3 function for
splitting a list of files into chunks

Response:
def splitfiles(data, raws, ipyclient):
    """ sends raws to be chunked"""

    ## create a tmpdir for chunked_files and a chunk optimizer
    tmpdir = os.path.join(data.paramsdict["project_dir"], "tmp-chunks-"+data.name)
    if os.path.exists(tmpdir):
        shutil.rmtree(tmpdir)
    os.makedirs(tmpdir)

    ## chunk into 8M reads
    totalreads = estimate_optim(data, raws[0][0], ipyclient)
    optim = int(8e6)
    njobs = int(totalreads/(optim/4.)) * len(raws)

    ## if more files than cpus: no chunking
    nosplit = 0
    if (len(raws) > len(ipyclient)) or (totalreads < optim):
        nosplit = 1

    ## send slices N at a time. The dict chunkfiles stores a tuple of rawpairs
    ## dictionary to store asyncresults for sorting jobs
    start = time.time()
    chunkfiles = {}
    for fidx, tups in enumerate(raws):
        handle = os.path.splitext(os.path.basename(tups[0]))[0]
        ## if number of lines is > 20M then just submit it
        if nosplit:
            chunkfiles[handle] = [tups]
        else:
            ## chunk the file using zcat_make_temps
            chunklist = zcat_make_temps(data, tups, fidx, tmpdir, optim, njobs, start)
            chunkfiles[handle] = chunklist
    if not nosplit:
        print("")

    return chunkfiles


Instruction:
Can you generate the documentation for the following Python 3 function
def concat_chunks(data, ipyclient):

    ## collate files progress bar
    start = time.time()
    printstr = ' writing/compressing   | {} | s1 |'
    lbview = ipyclient.load_balanced_view()
    elapsed = datetime.timedelta(seconds=int(time.time()-start))
    progressbar(10, 0, printstr.format(elapsed), spacer=data._spacer)
    ## get all the files
    ftmps = glob.glob(os.path.join(data.dirs.fastqs, "tmp_*.fastq"))

    ## a dict to assign tmp files to names/reads
    r1dict = {}
    r2dict = {}
    for sname in data.barcodes:
        if "-technical-replicate-" in sname:
            sname = sname.rsplit("-technical-replicate", 1)[0]
        r1dict[sname] = []
        r2dict[sname] = []

    ## assign to name keys
    for ftmp in ftmps:
        base, orient, _ = ftmp.rsplit("_", 2)
        sname = base.rsplit("/", 1)[-1].split("tmp_", 1)[1]
        if orient == "R1":
            r1dict[sname].append(ftmp)
        else:
            r2dict[sname].append(ftmp)

    ## concatenate files
    snames = []
    for sname in data.barcodes:
        if "-technical-replicate-" in sname:
            sname = sname.rsplit("-technical-replicate", 1)[0]
        snames.append(sname)

    writers = []
    for sname in set(snames):
        tmp1s = sorted(r1dict[sname])
        tmp2s = sorted(r2dict[sname])
        writers.append(lbview.apply(collate_files, *[data, sname, tmp1s, tmp2s]))

    total = len(writers)
    while 1:
        ready = [i.ready() for i in writers]
        elapsed = datetime.timedelta(seconds=int(time.time()-start))
        progressbar(total, sum(ready), printstr.format(elapsed), spacer=data._spacer)
        time.sleep(0.1)
        if all(ready):
            print("")
            break

Response: Concatenate chunks into one sample name file.


Instruction:
Create a Python 3 function for
submitting chunks to be sorted by the barmatch() function then calls putstats().

Response:
def demux2(data, chunkfiles, cutters, longbar, matchdict, ipyclient):
    """
    Submit chunks to be sorted by the barmatch() function then
    calls putstats().
    """

    ## parallel stuff, limit to 1/4 of available cores for RAM limits.
    start = time.time()
    printstr = ' sorting reads         | {} | s1 |'
    lbview = ipyclient.load_balanced_view(targets=ipyclient.ids[::4])

    ## store statcounters and async results in dicts
    perfile = {}
    filesort = {}
    total = 0
    done = 0

    ## chunkfiles is a dict with {handle: chunkslist, ...}. The func barmatch
    ## writes results to samplename files with PID number, and also writes a
    ## pickle for chunk specific results with fidx suffix, which it returns.
    for handle, rawtuplist in chunkfiles.items():
        ## get args for job
        for fidx, rawtuple in enumerate(rawtuplist):
            #handle = os.path.splitext(os.path.basename(rawtuple[0]))[0]
            args = (data, rawtuple, cutters, longbar, matchdict, fidx)

            ## submit the job
            async = lbview.apply(barmatch, *args)
            filesort[total] = (handle, async)
            total += 1

            ## get ready to receive stats: 'total', 'cutfound', 'matched'
            perfile[handle] = np.zeros(3, dtype=np.int)

    ## stats for each sample
    fdbars = {}
    fsamplehits = Counter()
    fbarhits = Counter()
    fmisses = Counter()
    ## a tuple to hold my dictionaries
    statdicts = perfile, fsamplehits, fbarhits, fmisses, fdbars

    ## wait for jobs to finish
    while 1:
        fin = [i for i, j in filesort.items() if j[1].ready()]
        #fin = [i for i in jobs if i[1].ready()]
        elapsed = datetime.timedelta(seconds=int(time.time()-start))
        progressbar(total, done, printstr.format(elapsed), spacer=data._spacer)
        time.sleep(0.1)

        ## should we break?
        if total == done:
            print("")
            break

        ## cleanup
        for key in fin:
            tup = filesort[key]
            if tup[1].successful():
                pfile = tup[1].result()
                handle = tup[0]
                if pfile:
                    ## check if this needs to return data
                    putstats(pfile, handle, statdicts)
                    ## purge to conserve memory
                    del filesort[key]
                    done += 1

    return statdicts


Instruction:
Write a Python 3 function for
submitting chunks to be sorted

Response:
def demux(data, chunkfiles, cutters, longbar, matchdict, ipyclient):
    """ submit chunks to be sorted """

    ## parallel stuff
    start = time.time()
    printstr = ' sorting reads         | {} | s1 |'
    lbview = ipyclient.load_balanced_view()

    ## store statcounters and async results in dicts
    perfile = {}
    filesort = {}
    for handle, rawtuplist in chunkfiles.items():
        ## get args for job
        for fidx, rawtuple in enumerate(rawtuplist):
            #handle = os.path.splitext(os.path.basename(rawtuple[0]))[0]
            args = (data, rawtuple, cutters, longbar, matchdict, fidx)

            ## submit the job
            filesort[handle] = lbview.apply(barmatch, *args)

            ## get ready to receive stats: 'total', 'cutfound', 'matched'
            perfile[handle] = np.zeros(3, dtype=np.int)

    ## stats for each sample
    fdbars = {}
    fsamplehits = Counter()
    fbarhits = Counter()
    fmisses = Counter()
    ## a tuple to hold my dictionaries
    statdicts = perfile, fsamplehits, fbarhits, fmisses, fdbars

    try:
        kbd = 0
        total = len(chunkfiles)
        done = 0
        ## wait for jobs to finish
        while 1:
            fin = [i for i, j in filesort.items() if j.ready()]
            elapsed = datetime.timedelta(seconds=int(time.time()-start))
            progressbar(total, done, printstr.format(elapsed), spacer=data._spacer)
            time.sleep(0.1)

            ## should we break?
            if total == done:
                print("")
                break

            ## cleanup
            for job in fin:
                if filesort[job].successful():
                    pfile = filesort[job].result()
                    #if result:
                    if pfile:
                        ## check if this needs to return data
                        putstats(pfile, handle, statdicts)

                        ## purge to conserve memory
                        del filesort[job]
                        done += 1

        ## keep tacking progreess during writing stage
        start = time.time()
        printstr = ' writing/compressing   | {} | s1 |'
        elapsed = datetime.timedelta(seconds=int(time.time()-start))
        progressbar(10, 0, printstr.format(elapsed), spacer=data._spacer)

    except KeyboardInterrupt:
        ## wait to cleanup
        kbd = 1
        raise

    ## only proceed here if barmatch jobs were not interrupted
    else:
        ## collate files and do progress bar
        ftmps = glob.glob(os.path.join(data.dirs.fastqs, "tmp_*.fastq"))

        ## a dict to assign tmp files to names/reads
        r1dict = {}
        r2dict = {}
        for sname in data.barcodes:
            r1dict[sname] = []
            r2dict[sname] = []

        ## assign to name keys
        for ftmp in ftmps:
            ## split names
            base, orient, _ = ftmp.rsplit("_", 2)
            sname = base.rsplit("/", 1)[-1].split("tmp_", 1)[1]
            ## put into dicts
            if orient == "R1":
                r1dict[sname].append(ftmp)
            else:
                r2dict[sname].append(ftmp)

        ## concatenate files
        total = len(data.barcodes)
        done = 0

        ## store asyncs of collate jobs
        writers = []
        for sname in data.barcodes:
            tmp1s = sorted(r1dict[sname])
            tmp2s = sorted(r2dict[sname])
            writers.append(lbview.apply(collate_files,
                           *[data, sname, tmp1s, tmp2s]))

        ## track progress of collate jobs
        while 1:
            ready = [i.ready() for i in writers]
            elapsed = datetime.timedelta(seconds=int(time.time()-start))
            progressbar(total, sum(ready), printstr.format(elapsed), spacer=data._spacer)
            time.sleep(0.1)
            if all(ready):
                print("")
                break

    finally:
        ## clean up junk files
        tmpfiles = glob.glob(os.path.join(data.dirs.fastqs, "tmp_*_R*.fastq"))
        tmpfiles += glob.glob(os.path.join(data.dirs.fastqs, "tmp_*.p"))
        for tmpf in tmpfiles:
            os.remove(tmpf)

        if kbd:
            raise KeyboardInterrupt()
        else:
            ## build stats from dictionaries
            perfile, fsamplehits, fbarhits, fmisses, fdbars = statdicts
            make_stats(data, perfile, fsamplehits, fbarhits, fmisses, fdbars)


Instruction:
Can you generate the documentation for the following Python 3 function
def putstats(pfile, handle, statdicts):

    ## load in stats
    with open(pfile, 'r') as infile:
        filestats, samplestats = pickle.load(infile)

    ## get dicts from statdicts tuple
    perfile, fsamplehits, fbarhits, fmisses, fdbars = statdicts

    ## pull new stats
    #handle = os.path.splitext(os.path.basename(handle))[0]
    perfile[handle] += filestats

    ## update sample stats
    samplehits, barhits, misses, dbars = samplestats
    fsamplehits.update(samplehits)
    fbarhits.update(barhits)
    fmisses.update(misses)
    fdbars.update(dbars)

    ## repack the tuple and return
    statdicts = perfile, fsamplehits, fbarhits, fmisses, fdbars
    return statdicts

Response: put stats from pickles into a dictionary


Instruction:
How would you explain what the following Python 3 function does
def zcat_make_temps(data, raws, num, tmpdir, optim, njobs, start):

    printstr = ' chunking large files  | {} | s1 |'

    ## split args
    tmpdir = os.path.realpath(tmpdir)
    LOGGER.info("zcat is using optim = %s", optim)

    ## read it, is it gzipped?
    catcmd = ["cat"]
    if raws[0].endswith(".gz"):
        catcmd = ["gunzip", "-c"]

    ## get reading commands for r1s, r2s
    cmd1 = catcmd + [raws[0]]
    cmd2 = catcmd + [raws[1]]

    ## second command splits and writes with name prefix
    cmd3 = ["split", "-a", "4", "-l", str(int(optim)), "-",
            os.path.join(tmpdir, "chunk1_"+str(num)+"_")]
    cmd4 = ["split", "-a", "4", "-l", str(int(optim)), "-",
            os.path.join(tmpdir, "chunk2_"+str(num)+"_")]

    ### run splitter
    proc1 = sps.Popen(cmd1, stderr=sps.STDOUT, stdout=sps.PIPE)
    proc3 = sps.Popen(cmd3, stderr=sps.STDOUT, stdout=sps.PIPE, stdin=proc1.stdout)

    ## wrap the actual call so we can kill it if anything goes awry
    while 1:
        try:
            if not isinstance(proc3.poll(), int):
                elapsed = datetime.timedelta(seconds=int(time.time()-start))
                done = len(glob.glob(os.path.join(tmpdir, 'chunk1_*')))
                progressbar(njobs, min(njobs, done), printstr.format(elapsed), spacer=data._spacer)
                time.sleep(0.1)
            else:
                res = proc3.communicate()[0]
                proc1.stdout.close()
                break

        except KeyboardInterrupt:
            proc1.kill()
            proc3.kill()
            raise KeyboardInterrupt()

    if proc3.returncode:
        raise IPyradWarningExit(" error in %s: %s", cmd3, res)

    ## grab output handles
    chunks1 = glob.glob(os.path.join(tmpdir, "chunk1_"+str(num)+"_*"))
    chunks1.sort()

    if "pair" in data.paramsdict["datatype"]:
        proc2 = sps.Popen(cmd2, stderr=sps.STDOUT, stdout=sps.PIPE)
        proc4 = sps.Popen(cmd4, stderr=sps.STDOUT, stdout=sps.PIPE, stdin=proc2.stdout)

        ## wrap the actual call so we can kill it if anything goes awry
        while 1:
            try:
                if not isinstance(proc4.poll(), int):
                    elapsed = datetime.timedelta(seconds=int(time.time()-start))
                    done = len(glob.glob(os.path.join(tmpdir, 'chunk1_*')))
                    progressbar(njobs, min(njobs, done), printstr.format(elapsed), data._spacer)
                    time.sleep(0.1)
                else:
                    res = proc4.communicate()[0]
                    proc2.stdout.close()
                    break

            except KeyboardInterrupt:
                proc2.kill()
                proc4.kill()
                raise KeyboardInterrupt()

        if proc4.returncode:
            raise IPyradWarningExit(" error in %s: %s", cmd4, res)

        ## grab output handles
        chunks2 = glob.glob(os.path.join(tmpdir, "chunk2_"+str(num)+"_*"))
        chunks2.sort()

    else:
        chunks2 = [0]*len(chunks1)

    assert len(chunks1) == len(chunks2), \
        "R1 and R2 files are not the same length."

    ## ensure full progress bar b/c estimates njobs could be off
    progressbar(10, 10, printstr.format(elapsed), spacer=data._spacer)
    return zip(chunks1, chunks2)

Response: Create large files from a list of files.


Instruction:
Can you tell what is the following Python 3 function doing
def _plotshare(share, names, **kwargs):

    ## set the colormap
    colormap = toyplot.color.LinearMap(toyplot.color.brewer.palette("Spectral"),
                                 domain_min=share.min(), domain_max=share.max())

    ## set up canvas
    if not kwargs.get('width'):
        width=900
    else:
        width = kwargs['width']
    canvas = toyplot.Canvas(width=width, height=width*0.77778)

    ## order the dta
    table = canvas.matrix((share, colormap),
                          bounds=(50, canvas.height-100,
                                  50, canvas.height-100),
                          step=5, tshow=False, lshow=False)

    ## put a box around the table
    table.body.grid.vlines[..., [0, -1]] = 'single'
    table.body.grid.hlines[[0, -1], ...] = 'single'

    ## make hover info on grid
    for i, j in itertools.product(range(len(share)), repeat=2):
        table.body.cell(i,j).title = "%s, %s : %s" % (names[i], names[j], int(share[i,j]))

    ## create barplot
    axes = canvas.cartesian(bounds=(665, 800, 90, 560))

    ## make a hover for barplot
    zf = zip(names[::-1], share.diagonal()[::-1])
    barfloater = ["%s: %s" % (i, int(j)) for i, j in zf]

    ## plot bars
    axes.bars(share.diagonal()[::-1], along='y', title=barfloater)

    ## hide spine, move labels to the left,
    ## use taxon names, rotate angle, align
    axes.y.spine.show = False
    axes.y.ticks.labels.offset = 0
    axes.y.ticks.locator = toyplot.locator.Explicit(range(len(names)),
                                                    labels=names[::-1])
    axes.y.ticks.labels.angle = -90
    axes.y.ticks.labels.style = {"baseline-shift":0,
                                 "text-anchor":"end",
                                 "font-size":"8px"}

    ## rotate xlabels, align with ticks, change to thousands, move up on canvas
    ## show ticks, and hide popup coordinates
    axes.x.ticks.labels.angle = 90
    axes.x.ticks.labels.offset = 20
    axes.x.ticks.locator = toyplot.locator.Explicit(
        range(0, int(share.max()),
                 int(share.max() / 10)),
        ["{}".format(i) for i in range(0, int(share.max()),
                                           int(share.max() / 10))])
    axes.x.ticks.labels.style = {"baseline-shift":0,
                                 "text-anchor":"end",
                                 "-toyplot-anchor-shift":"15px"}
    axes.x.ticks.show = True

    ## add labels
    label_style = {"font-size": "16px", "font-weight": "bold"}
    canvas.text(300, 60, "Matrix of shared RAD loci", style=label_style)
    canvas.text(700, 60, "N RAD loci per sample", style=label_style)

    return canvas, axes

Response: make toyplot matrix fig


Instruction:
Write a Python 3 script to
parse the loci file list and return presence and absence matrix

Response:
def _getarray(loci, tree):
    """
    parse the loci file list and return presence/absence matrix
    ordered by the tips on the tree
    """

    ## order tips
    tree.ladderize()

    ## get tip names
    snames = tree.get_leaf_names()

    ## make an empty matrix
    lxs = np.zeros((len(snames), len(loci)), dtype=np.int)

    ## fill the matrix
    for loc in xrange(len(loci)):
        for seq in loci[loc].split("\n")[:-1]:
            lxs[snames.index(seq.split()[0]), loc] += 1

    return lxs, snames


Instruction:
How would you implement a function in Python 3 that
fills a matrix with pairwise data sharing

Response:
def _countmatrix(lxs):
    """ fill a matrix with pairwise data sharing """

    ## an empty matrix
    share = np.zeros((lxs.shape[0], lxs.shape[0]))

    ## fill above
    names = range(lxs.shape[0])
    for row in lxs:
        for samp1, samp2 in itertools.combinations(names, 2):
            shared = lxs[samp1, lxs[samp2] > 0].sum()
            share[samp1, samp2] = shared

    ## mirror below
    ##share[]

    ## fill diagonal with total sample coverage
    for row in xrange(len(names)):
        share[row, row] = lxs[row].sum()

    return share