Instruction:
Can you tell what is the following Python 3 function doing
def paramname(param=""):

    try:
        name = pinfo[str(param)][0].strip().split(" ")[1]
    except (KeyError, ValueError) as err:
        ## TODO: paramsinfo get description by param string not working.
        ## It would be cool to have an assembly object bcz then you could
        ## just do this:
        ##
        ## print(pinfo[data.paramsinfo.keys().index(param)])
        print("\tKey name/number not recognized - ".format(param), err)
        raise

    return name

Response: Get the param name from the dict index value.


Instruction:
Write a Python 3 script for
returning detailed information for a parameter.

Response:
def paraminfo(param="", short=False):
    """ Returns detailed information for the numbered parameter.
        Further information is available in the tutorial.
        Unlike params() this function doesn't deal well with *
        It only takes one parameter at a time and returns the desc
    """

    ## If the short flag is set return the short description, otherwise
    ## return the long.
    if short:
        desc = 1
    else:
        desc = 0

    try:
        description = pinfo[str(param)][desc]
    except (KeyError, ValueError) as err:
        ## TODO: paramsinfo get description by param string not working.
        ## It would be cool to have an assembly object bcz then you could
        ## just do this:
        ##
        ## print(pinfo[data.paramsinfo.keys().index(param)])
        print("\tKey name/number not recognized - ".format(param), err)
        raise

    return description


Instruction:
Here you have a function in Python 3, explain what it does
def paramsinfo(param="", short=False):
    if short:
        desc = 1
    else:
        desc = 0

    if param == "*":
        for key in pinfo:
            print(pinfo[str(key)][desc])
    elif param:
        try:
            print(pinfo[str(param)][desc])
        except (KeyError, ValueError) as err:
            ## TODO: paramsinfo get description by param string not working.
            ## It would be cool to have an assembly object bcz then you could
            ## just do this:
            ##
            ## print(pinfo[data.paramsinfo.keys().index(param)])
            print("\tKey name/number not recognized", err)
            raise
    else:
        print("Enter a name or number for explanation of the parameter\n")
        for key in pinfo:
            print(pinfo[str(key)][desc].split("\n")[1][2:-10])

Response: This function prints out the information about the parameter.


Instruction:
Make a summary of the following Python 3 code
def update_assembly(data):

    print("##############################################################")
    print("Updating assembly to current version")
    ## New assembly object to update pdate from.
    new_assembly = ip.Assembly("update", quiet=True)

    ## Hackersonly dict gets automatically overwritten
    ## Always use the current version for params in this dict.
    data._hackersonly = deepcopy(new_assembly._hackersonly)

    new_params = set(new_assembly.paramsdict.keys())

    my_params = set(data.paramsdict.keys())

    ## Find all params in loaded assembly that aren't in the new assembly.
    ## Make a new dict that doesn't include anything in removed_params
    removed_params = my_params.difference(new_params)
    for i in removed_params:
        print("Removing parameter: {} = {}".format(i, data.paramsdict[i]))

    ## Find all params that are in the new paramsdict and not in the old one.
    ## If the set isn't emtpy then we create a new dictionary based on the new
    ## assembly parameters and populated with currently loaded assembly values.
    ## Conditioning on not including any removed params. Magic.
    added_params = new_params.difference(my_params)
    for i in added_params:
        print("Adding parameter: {} = {}".format(i, new_assembly.paramsdict[i]))

    print("\nPlease take note of these changes. Every effort is made to\n"\
            +"ensure compatibility across versions of ipyrad. See online\n"\
            +"documentation for further details about new parameters.")
    time.sleep(5)
    print("##############################################################")

    if added_params:
        for i in data.paramsdict:
            if i not in removed_params:
                new_assembly.paramsdict[i] = data.paramsdict[i]
        data.paramsdict = deepcopy(new_assembly.paramsdict)

    data.save()
    return data

Response: Update the current assembly with the new parameters.


Instruction:
Write a Python 3 function for
saving assembly and samples as json

Response:
def save_json(data):
    """ Save assembly and samples as json """

    ## data as dict
    #### skip _ipcluster because it's made new
    #### skip _headers because it's loaded new
    #### statsfiles save only keys
    #### samples save only keys
    datadict = OrderedDict([
        ("_version", data.__dict__["_version"]),
        ("_checkpoint", data.__dict__["_checkpoint"]),
        ("name", data.__dict__["name"]),
        ("dirs", data.__dict__["dirs"]),
        ("paramsdict", data.__dict__["paramsdict"]),
        ("samples", data.__dict__["samples"].keys()),
        ("populations", data.__dict__["populations"]),
        ("database", data.__dict__["database"]),
        ("clust_database", data.__dict__["clust_database"]),
        ("outfiles", data.__dict__["outfiles"]),
        ("barcodes", data.__dict__["barcodes"]),
        ("stats_files", data.__dict__["stats_files"]),
        ("_hackersonly", data.__dict__["_hackersonly"]),
        ])

    ## sample dict
    sampledict = OrderedDict([])
    for key, sample in data.samples.iteritems():
        sampledict[key] = sample._to_fulldict()

    ## json format it using cumstom Encoder class
    fulldumps = json.dumps({
        "assembly": datadict,
        "samples": sampledict
        },
        cls=Encoder,
        sort_keys=False, indent=4, separators=(",", ":"),
        )

    ## save to file
    assemblypath = os.path.join(data.dirs.project, data.name+".json")
    if not os.path.exists(data.dirs.project):
        os.mkdir(data.dirs.project)

    ## protect save from interruption
    done = 0
    while not done:
        try:
            with open(assemblypath, 'w') as jout:
                jout.write(fulldumps)
            done = 1
        except (KeyboardInterrupt, SystemExit):
            print('.')
            continue


Instruction:
Explain what the following Python 3 code does
def load_json(path, quiet=False, cli=False):

    ## load the JSON string and try with name+.json
    checkfor = [path+".json", path]
    for inpath in checkfor:
        inpath = inpath.replace("~", os.path.expanduser("~"))
        try:
            with open(inpath, 'r') as infile:
                ## uses _tup_and_byte to ensure ascii and tuples are correct
                fullj = json.loads(infile.read(), object_hook=_tup_and_byte)
        except IOError:
            pass

    ## create a new empty Assembly
    try:
        oldname = fullj["assembly"].pop("name")
        olddir = fullj["assembly"]["dirs"]["project"]
        oldpath = os.path.join(olddir, os.path.splitext(oldname)[0]+".json")
        null = ip.Assembly(oldname, quiet=True, cli=cli)

    except (UnboundLocalError, AttributeError) as inst:
        raise IPyradWarningExit("""
    Could not find saved Assembly file (.json) in expected location.
    Checks in: [project_dir]/[assembly_name].json
    Checked: {}
    """.format(inpath))

    ## print msg with shortpath
    if not quiet:
        oldpath = oldpath.replace(os.path.expanduser("~"), "~")
        print("{}loading Assembly: {}".format(null._spacer, oldname))
        print("{}from saved path: {}".format(null._spacer, oldpath))

    ## First get the samples. Create empty sample dict of correct length
    samplekeys = fullj["assembly"].pop("samples")
    null.samples = {name: "" for name in samplekeys}

    ## Next get paramsdict and use set_params to convert values back to
    ## the correct dtypes. Allow set_params to fail because the object will
    ## be subsequently updated by the params from the params file, which may
    ## correct any errors/incompatibilities in the old params file
    oldparams = fullj["assembly"].pop("paramsdict")
    for param, val in oldparams.iteritems():
        ## a fix for backward compatibility with deprecated options
        if param not in ["assembly_name", "excludes", "outgroups"]:
            try:
                null.set_params(param, val)
            except IPyradWarningExit as inst:
                #null.set_params(param, "")
                LOGGER.warning("""
    Load assembly error setting params. Not critical b/c new params file may
    correct the problem. Recorded here for debugging:
    {}
    """.format(inst))

    ## Import the hackersonly dict. In this case we don't have the nice
    ## set_params so we're shooting from the hip to reset the values
    try:
        oldhackersonly = fullj["assembly"].pop("_hackersonly")
        for param, val in oldhackersonly.iteritems():
            if val == None:
                null._hackersonly[param] = None
            else:
                null._hackersonly[param] = val

    except Exception as inst:
        LOGGER.warning("""
    Load assembly error resetting hackersonly dict element. We will just use
    the default value in the current assembly.""")

    ## Check remaining attributes of Assembly and Raise warning if attributes
    ## do not match up between old and new objects
    newkeys = null.__dict__.keys()
    oldkeys = fullj["assembly"].keys()

    ## find shared keys and deprecated keys
    sharedkeys = set(oldkeys).intersection(set(newkeys))
    lostkeys = set(oldkeys).difference(set(newkeys))

    ## raise warning if there are lost/deprecated keys
    if lostkeys:
        LOGGER.warning("""
    load_json found {a} keys that are unique to the older Assembly.
        - assembly [{b}] v.[{c}] has: {d}
        - current assembly is v.[{e}]
        """.format(a=len(lostkeys),
                   b=oldname,
                   c=fullj["assembly"]["_version"],
                   d=lostkeys,
                   e=null._version))

    ## load in remaining shared Assembly attributes to null
    for key in sharedkeys:
        null.__setattr__(key, fullj["assembly"][key])

    ## load in svd results if they exist
    try:
        if fullj["assembly"]["svd"]:
            null.__setattr__("svd", fullj["assembly"]["svd"])
            null.svd = ObjDict(null.svd)
    except Exception:
        LOGGER.debug("skipping: no svd results present in old assembly")

    ## Now, load in the Sample objects json dicts
    sample_names = fullj["samples"].keys()
    if not sample_names:
        raise IPyradWarningExit("""
    No samples found in saved assembly. If you are just starting a new
    assembly the file probably got saved erroneously, so it's safe to try
    removing the assembly file (e.g., rm {}.json) and restarting.

    If you fully completed step 1 and you see this message you should probably
    contact the developers.
    """.format(inpath))

    sample_keys = fullj["samples"][sample_names[0]].keys()
    stats_keys = fullj["samples"][sample_names[0]]["stats"].keys()
    stats_dfs_keys = fullj["samples"][sample_names[0]]["stats_dfs"].keys()
    ind_statkeys = \
        [fullj["samples"][sample_names[0]]["stats_dfs"][i].keys() \
        for i in stats_dfs_keys]
    ind_statkeys = list(itertools.chain(*ind_statkeys))

    ## check against a null sample
    nsamp = ip.Sample()
    newkeys = nsamp.__dict__.keys()
    newstats = nsamp.__dict__["stats"].keys()
    newstatdfs = nsamp.__dict__["stats_dfs"].keys()
    newindstats = [nsamp.__dict__["stats_dfs"][i].keys() for i in newstatdfs]
    newindstats = list(itertools.chain(*[i.values for i in newindstats]))

    ## different in attributes?
    diffattr = set(sample_keys).difference(newkeys)
    diffstats = set(stats_keys).difference(newstats)
    diffindstats = set(ind_statkeys).difference(newindstats)

    ## Raise warning if any oldstats were lost or deprecated
    alldiffs = diffattr.union(diffstats).union(diffindstats)
    if any(alldiffs):
        LOGGER.warning("""
    load_json found {a} keys that are unique to the older Samples.
        - assembly [{b}] v.[{c}] has: {d}
        - current assembly is v.[{e}]
        """.format(a=len(alldiffs),
                   b=oldname,
                   c=fullj["assembly"]["_version"],
                   d=alldiffs,
                   e=null._version))

    ## save stats and statsfiles to Samples
    for sample in null.samples:
        ## create a null Sample
        null.samples[sample] = ip.Sample()

        ## save stats
        sdat = fullj["samples"][sample]['stats']
        ## Reorder the keys so they ascend by step, only include
        ## stats that are actually in the sample. newstats is a
        ## list of the new sample stat names, and stats_keys
        ## are the names of the stats from the json file.
        newstats = [x for x in newstats if x in stats_keys]
        null.samples[sample].stats = pd.Series(sdat).reindex(newstats)

        ## save stats_dfs
        for statskey in stats_dfs_keys:
            null.samples[sample].stats_dfs[statskey] = \
                pd.Series(fullj["samples"][sample]["stats_dfs"][statskey])\
                .reindex(nsamp.__dict__["stats_dfs"][statskey].keys())

        ## save Sample files
        for filehandle in fullj["samples"][sample]["files"].keys():
            null.samples[sample].files[filehandle] = \
                fullj["samples"][sample]["files"][filehandle]

    ## build the Assembly object stats_dfs
    for statskey in stats_dfs_keys:
        indstat = null._build_stat(statskey)
        if not indstat.empty:
            null.stats_dfs[statskey] = indstat

    ## add remaning attributes to null Samples
    shared_keys = set(sample_keys).intersection(newkeys)
    shared_keys.discard("stats")
    shared_keys.discard("files")
    shared_keys.discard("stats_files")
    shared_keys.discard("stats_dfs")

    for sample in null.samples:
        ## set the others
        for key in shared_keys:
            null.samples[sample].__setattr__(key, fullj["samples"][sample][key])

    ## ensure objects are object dicts
    null.dirs = ObjDict(null.dirs)
    null.stats_files = ObjDict(null.stats_files)
    null.stats_dfs = ObjDict(null.stats_dfs)
    null.populations = ObjDict(null.populations)
    null.outfiles = ObjDict(null.outfiles)

    return null

Response: Load a json serialized Assembly object and ensure it matches to the current INDRA Assembly object


Instruction:
Can you write a function in Python 3 where it
functions to encode json string

Response:
def encode(self, obj):
        """ function to encode json string"""
        def hint_tuples(item):
            """ embeds __tuple__ hinter in json strings """
            if isinstance(item, tuple):
                return {'__tuple__': True, 'items': item}
            if isinstance(item, list):
                return [hint_tuples(e) for e in item]
            if isinstance(item, dict):
                return {
                    key: hint_tuples(val) for key, val in item.iteritems()
                    }
            else:
                return item

        return super(Encoder, self).encode(hint_tuples(obj))


Instruction:
Make a summary of the following Python 3 code
def baba_panel_plot(
    ttree,
    tests,
    boots,
    show_tip_labels=True,
    show_test_labels=True,
    use_edge_lengths=False,
    collapse_outgroup=False,
    pct_tree_x=0.4,
    pct_tree_y=0.2,
    alpha=3.0,
    *args,
    **kwargs):
    """
    signature...
    """

    ## create Panel plot object and set height & width
    bootsarr = np.array(boots)
    panel = Panel(ttree, tests, bootsarr, alpha)
    if not kwargs.get("width"):
        panel.kwargs["width"] = min(1000, 50*len(panel.tree))
    if not kwargs.get("height"):
        panel.kwargs["height"] = min(1000, 50*len(panel.tests))

    ## update defaults with kwargs & update size based on ntips & ntests
    kwargs.update(dict(pct_tree_x=pct_tree_x, pct_tree_y=pct_tree_y))
    panel.kwargs.update(kwargs)

    ## create a canvas and a single cartesian coord system
    canvas = toyplot.Canvas(height=panel.kwargs['height'], width=panel.kwargs['width'])
    axes = canvas.cartesian(bounds=("10%", "90%", "5%", "95%"))
    axes.show = False

    ## add panels to axes
    panel.panel_tree(axes)
    panel.panel_test(axes)
    panel.panel_tip_labels(axes)
    if isinstance(boots, np.ndarray):
        panel.panel_results(axes)
    return canvas, axes, panel

Response: Create a panel plot object and return it


Instruction:
Explain what the following Python 3 code does
def depthplot(data, samples=None, dims=(None,None), canvas=(None,None),
              xmax=50, log=False, outprefix=None, use_maxdepth=False):
    """ plots histogram of coverages across clusters"""

    ## select samples to be plotted, requires depths info
    if not samples:
        samples = data.samples.keys()
        samples.sort()
    subsamples = OrderedDict([(i, data.samples[i]) for i in samples])

    ## get canvas dimensions based on n-samples
    if any(dims):
        ## user-supplied dimensions (...)
        print("userdims")
    else:
        if len(subsamples) <= 4:
            ## set dimension to N samples
            dims = (1, len(subsamples))
        else:
            dims = (len(subsamples)/4, 4)

    ## create canvas
    if any(canvas):
        print("usercanvas")
        canvas = toyplot.Canvas(width=canvas[0], height=canvas[1])
    else:
        canvas = toyplot.Canvas(width=200*dims[1], height=150*dims[0])

    ## get all of the data arrays
    for panel, sample in enumerate(subsamples):
        ## statistical called bins
        statdat = subsamples[sample].depths
        statdat = statdat[statdat >= data.paramsdict["mindepth_statistical"]]
        if use_maxdepth:
            statdat = {i:j for (i, j) in statdat if \
                                        i < data.paramsdict["maxdepth"]}
        sdat = np.histogram(statdat, range(50))

        ## majrule called bins
        statdat = subsamples[sample].depths
        statdat = statdat[statdat < data.paramsdict["mindepth_statistical"]]
        statdat = statdat[statdat >= data.paramsdict["mindepth_majrule"]]
        if use_maxdepth:
            statdat = statdat[statdat < data.paramsdict["maxdepth"]]
        mdat = np.histogram(statdat, range(50))

        ## excluded bins
        tots = data.samples[sample].depths
        tots = tots[tots < data.paramsdict["mindepth_majrule"]]
        if use_maxdepth:
            tots = tots[tots < data.paramsdict["maxdepth"]]
        edat = np.histogram(tots, range(50))

        ## fill in each panel of canvas with a sample
        axes = canvas.cartesian(grid=(dims[0], dims[1], panel), gutter=25)
        axes.x.domain.xmax = xmax
        axes.label.text = sample
        if log:
            axes.y.scale = "log"

        # heights = np.column_stack((sdat,mdat,edat))
        axes.bars(sdat)
        axes.bars(edat)
        axes.bars(mdat)

    ## return objects to be saved...
    if outprefix:
        toyplot.html.render(canvas, fobj=outprefix+".html")
        toyplot.svg.render(canvas, fobj=outprefix+".svg")

Response: plot histogram of coverages across clusters


Instruction:
Implement a Python 3 function for
parsing the 00 outfile as a pandas DataFrame

Response:
def _parse_00(ofile):
    """
    return 00 outfile as a pandas DataFrame
    """
    with open(ofile) as infile:
        ## read in the results summary from the end of the outfile
        arr = np.array(
            [" "] + infile.read().split("Summary of MCMC results\n\n\n")[1:][0]\
            .strip().split())

        ## reshape array
        rows = 12
        cols = (arr.shape[0] + 1) / rows
        arr = arr.reshape(rows, cols)

        ## make into labeled data frame
        df = pd.DataFrame(
            data=arr[1:, 1:],
            columns=arr[0, 1:],
            index=arr[1:, 0],
            ).T
        return df


Instruction:
How would you implement a function in Python 3 that
loads existing results files for an object with this workdir and name.

Response:
def _load_existing_results(self, name, workdir):
        """
        Load existing results files for an object with this workdir and name.
        This does NOT reload the parameter settings for the object...
        """
        ## get mcmcs
        path = os.path.realpath(os.path.join(self.workdir, self.name))
        mcmcs = glob.glob(path+"_r*.mcmc.txt")
        outs = glob.glob(path+"_r*.out.txt")
        trees = glob.glob(path+"_r*.tre")

        for mcmcfile in mcmcs:
            if mcmcfile not in self.files.mcmcfiles:
                self.files.mcmcfiles.append(mcmcfile)
        for outfile in outs:
            if outfile not in self.files.outfiles:
                self.files.outfiles.append(outfile)
        for tree in trees:
            if tree not in self.files.treefiles:
                self.files.treefiles.append(tree)


Instruction:
Write a Python 3 function for
submitting bpp jobs to run on a cluster (ipyparallel Client). The seed for the random number generator if not set is randomly drawn, and if multiple reps are submitted (nreps>1) then each will draw a subsequent random seeds after that. An ipyclient connection is required. Asynchronous result objects are stored in the bpp object submitting the jobs. Parameters: ----------- nreps (int): submits nreps replicate jobs to the cluster each with a different random seed drawn starting from the starting seed. ipyclient (ipyparallel.Client) an ipyparallel.Client object connected to a running cluster. quiet (bool): whether to print that the jobs have been submitted randomize_order (bool): if True then when maxloci is set this will randomly sample a different set of N loci in each replicate, rather than sampling just the first N loci < maxloci. force (bool): Overwrite existing files with the same name. Default=False, skip over existing files.

Response:
def run(self,
        ipyclient,
        nreps=1,
        quiet=False,
        randomize_order=False,
        force=False,
        ):
        """
        Submits bpp jobs to run on a cluster (ipyparallel Client).
        The seed for the random number generator if not set is randomly
        drawn, and if multiple reps are submitted (nreps>1) then each will
        draw a subsequent random seeds after that. An ipyclient connection
        is required. Asynchronous result objects are stored in the bpp
        object submitting the jobs.

        Parameters:
        -----------
        nreps (int):
            submits nreps replicate jobs to the cluster each with a different
            random seed drawn starting from the starting seed.
        ipyclient (ipyparallel.Client)
            an ipyparallel.Client object connected to a running cluster.
        quiet (bool):
            whether to print that the jobs have been submitted
        randomize_order (bool):
            if True then when maxloci is set this will randomly sample a
            different set of N loci in each replicate, rather than sampling
            just the first N loci < maxloci.
        force (bool):
            Overwrite existing files with the same name. Default=False, skip
            over existing files.
        """

        ## is this running algorithm 00?
        is_alg00 = (not self.params.infer_sptree) and (not self.params.infer_delimit)

        ## clear out pre-existing files for this object
        self.files.mcmcfiles = []
        self.files.outfiles = []
        self.files.treefiles = []
        self.asyncs = []

        ## initiate random seed
        np.random.seed(self.params.seed)

        ## load-balancer
        lbview = ipyclient.load_balanced_view()

        ## send jobs
        for job in xrange(nreps):

            ## make repname and make ctl filename
            self._name = "{}_r{}".format(self.name, job)
            ctlhandle = os.path.realpath(
                os.path.join(self.workdir, "{}.ctl.txt".format(self._name)))

            ## skip if ctlfile exists
            if (not force) and (os.path.exists(ctlhandle)):
                print("Named ctl file already exists. Use force=True to" \
                    +" overwrite\nFilename:{}".format(ctlhandle))
            else:
                ## change seed and ctl for each rep, this writes into the ctl
                ## file the correct name for the other files which share the
                ## same rep number in their names.
                #self.params._seed = np.random.randint(0, 1e9, 1)[0]
                self._write_mapfile()
                #if randomize_order:
                self._write_seqfile(randomize_order=randomize_order)
                ctlfile = self._write_ctlfile()

                ## submit to engines
                async = lbview.apply(_call_bpp, *(self._kwargs["binary"], ctlfile, is_alg00))
                self.asyncs.append(async)

                ## save tree file if alg 00
                if is_alg00:
                    self.files.treefiles.append(
                        ctlfile.rsplit(".ctl.txt", 1)[0] + ".tre")

        if self.asyncs and (not quiet):
            sys.stderr.write("submitted {} bpp jobs [{}] ({} loci)\n"\
                             .format(nreps, self.name, self._nloci))


Instruction:
Implement a function in Python 3 to
write the bpp files for the current job to the working directory.

Response:
def write_bpp_files(self, randomize_order=False, quiet=False):
        """
        Writes bpp files (.ctl, .seq, .imap) to the working directory.

        Parameters:
        ------------
        randomize_order (bool):
            whether to randomize the locus order, this will allow you to
            sample different subsets of loci in different replicates when
            using the filters.maxloci option.
        quiet (bool):
            whether to print info to stderr when finished.
        """

        ## remove any old jobs with this same job name
        self._name = self.name
        oldjobs = glob.glob(os.path.join(self.workdir, self._name+"*.ctl.txt"))
        for job in oldjobs:
            os.remove(job)

        ## check params types
        ## ...

        ## write tmp files for the job
        self._write_seqfile(randomize_order=randomize_order)
        self._write_mapfile()#name=True)
        self._write_ctlfile()

        if not quiet:
            sys.stderr.write("input files created for job {} ({} loci)\n"\
                             .format(self._name, self._nloci))


Instruction:
Given the following Python 3 function, write the documentation
def _write_ctlfile(self):#, rep=None):

        ## A string to store ctl info
        ctl = []

        ## write the top header info
        ctl.append("seed = {}".format(self.params.seed))
        ctl.append("seqfile = {}".format(self.seqfile))
        ctl.append("Imapfile = {}".format(self.mapfile))

        path = os.path.realpath(os.path.join(self.workdir, self._name))
        mcmcfile = "{}.mcmc.txt".format(path)
        outfile = "{}.out.txt".format(path)
        if mcmcfile not in self.files.mcmcfiles:
            self.files.mcmcfiles.append(mcmcfile)
        if outfile not in self.files.outfiles:
            self.files.outfiles.append(outfile)

        ctl.append("mcmcfile = {}".format(mcmcfile))
        ctl.append("outfile = {}".format(outfile))

        ## number of loci (checks that seq file exists and parses from there)
        ctl.append("nloci = {}".format(self._nloci))
        ctl.append("usedata = {}".format(self.params.usedata))
        ctl.append("cleandata = {}".format(self.params.cleandata))

        ## infer species tree
        if self.params.infer_sptree:
            ctl.append("speciestree = 1 0.4 0.2 0.1")
        else:
            ctl.append("speciestree = 0")

        ## infer delimitation (with algorithm 1 by default)
        ctl.append("speciesdelimitation = {} {} {}"\
                   .format(self.params.infer_delimit,
                           self.params.delimit_alg[0],
                           " ".join([str(i) for i in self.params.delimit_alg[1:]])
                           )
                   )
        ## get tree values
        nspecies = str(len(self.imap))
        species = " ".join(sorted(self.imap))
        ninds = " ".join([str(len(self.imap[i])) for i in sorted(self.imap)])
        ctl.append(SPECIESTREE.format(nspecies, species, ninds, self.tree.write(format=9)))

        ## priors
        ctl.append("thetaprior = {} {}".format(*self.params.thetaprior))
        ctl.append("tauprior = {} {} {}".format(*self.params.tauprior))

        ## other values, fixed for now
        ctl.append("finetune = 1: {}".format(" ".join([str(i) for i in self.params.finetune])))
        #CTL.append("finetune = 1: 1 0.002 0.01 0.01 0.02 0.005 1.0")
        ctl.append("print = 1 0 0 0")
        ctl.append("burnin = {}".format(self.params.burnin))
        ctl.append("sampfreq = {}".format(self.params.sampfreq))
        ctl.append("nsample = {}".format(self.params.nsample))

        ## write out the ctl file
        ctlhandle = os.path.realpath(
                "{}.ctl.txt".format(os.path.join(self.workdir, self._name)))
        # if isinstance(rep, int):
        #     ctlhandle = os.path.realpath(
        #         "{}-r{}.ctl.txt".format(os.path.join(self.workdir, self._name), rep))
        # else:
        #     ctlhandle = os.path.realpath(
        #         "{}.ctl.txt".format(os.path.join(self.workdir, self._name)))
        with open(ctlhandle, 'w') as out:
            out.write("\n".join(ctl))

        return ctlhandle

Response: Write the ctl file.


Instruction:
Given the following Python 3 function, write the documentation
def copy(self, name, load_existing_results=False):

        ## make deepcopy of self.__dict__ but do not copy async objects
        subdict = {i:j for i,j in self.__dict__.iteritems() if i != "asyncs"}
        newdict = copy.deepcopy(subdict)

        ## make back into a bpp object
        if name == self.name:
            raise Exception("new object must have a different 'name' than its parent")
        newobj = Bpp(
            name=name,
            data=newdict["files"].data,
            workdir=newdict["workdir"],
            guidetree=newdict["tree"].write(),
            imap={i:j for i, j in newdict["imap"].items()},
            copied=True,
            load_existing_results=load_existing_results,
            )

        ## update special dict attributes but not files
        for key, val in newobj.params.__dict__.iteritems():
            newobj.params.__setattr__(key, self.params.__getattribute__(key))
        for key, val in newobj.filters.__dict__.iteritems():
            newobj.filters.__setattr__(key, self.filters.__getattribute__(key))

        ## new object must have a different name than it's parent
        return newobj

Response:
Returns a copy of the object with the same parameter settings
        but with the files. cmcfiles and files. outfiles attributes cleared and with a new name attribute.


Instruction:
Write a Python 3 function for
printing a summarized table of results from replicate runs or a list of separate TCOs.

Response:
def summarize_results(self, individual_results=False):
        """
        Prints a summarized table of results from replicate runs, or,
        if individual_result=True, then returns a list of separate
        dataframes for each replicate run.
        """

        ## return results depending on algorithm

        ## algorithm 00
        if (not self.params.infer_delimit) & (not self.params.infer_sptree):
            if individual_results:
                ## return a list of parsed CSV results
                return [_parse_00(i) for i in self.files.outfiles]
            else:
                ## concatenate each CSV and then get stats w/ describe
                return pd.concat(
                    [pd.read_csv(i, sep='\t', index_col=0) \
                    for i in self.files.mcmcfiles]).describe().T

        ## algorithm 01
        if self.params.infer_delimit & (not self.params.infer_sptree):
            return _parse_01(self.files.outfiles, individual=individual_results)

        ## others
        else:
            return "summary function not yet ready for this type of result"


Instruction:
Can you tell what is the following Python 3 function doing
def persistent_popen_align3(data, samples, chunk):

    ## data are already chunked, read in the whole thing
    with open(chunk, 'rb') as infile:
        clusts = infile.read().split("//\n//\n")[:-1]

    ## snames to ensure sorted order
    samples.sort(key=lambda x: x.name)
    snames = [sample.name for sample in samples]

    ## make a tmparr to store metadata (this can get huge, consider using h5)
    maxlen = data._hackersonly["max_fragment_length"] + 20
    indels = np.zeros((len(samples), len(clusts), maxlen), dtype=np.bool_)
    duples = np.zeros(len(clusts), dtype=np.bool_)

    ## create a persistent shell for running muscle in.
    proc = sps.Popen(["bash"],
                    stdin=sps.PIPE,
                    stdout=sps.PIPE,
                    universal_newlines=True)

    ## iterate over clusters until finished
    allstack = []
    #istack = []
    for ldx in xrange(len(clusts)):
        ## new alignment string for read1s and read2s
        aligned = []
        istack = []
        lines = clusts[ldx].strip().split("\n")
        names = lines[::2]
        seqs = lines[1::2]
        align1 = ""
        align2 = ""

        ## we don't allow seeds with no hits to make it here, currently
        #if len(names) == 1:
        #    aligned.append(clusts[ldx].replace(">", "").strip())

        ## find duplicates and skip aligning but keep it for downstream.
        if len(names) != len(set([x.rsplit("_", 1)[0] for x in names])):
            duples[ldx] = 1
            istack = ["{}\n{}".format(i[1:], j) for i, j in zip(names, seqs)]
            #aligned.append(clusts[ldx].replace(">", "").strip())

        else:
            ## append counter to names because muscle doesn't retain order
            names = [">{};*{}".format(j[1:], i) for i, j in enumerate(names)]

            try:
                ## try to split names on nnnn splitter
                clust1, clust2 = zip(*[i.split("nnnn") for i in seqs])

                ## make back into strings
                cl1 = "\n".join(itertools.chain(*zip(names, clust1)))
                cl2 = "\n".join(itertools.chain(*zip(names, clust2)))

                ## store allele (lowercase) info
                shape = (len(seqs), max([len(i) for i in seqs]))
                arrseqs = np.zeros(shape, dtype="S1")
                for row in range(arrseqs.shape[0]):
                    seqsrow = seqs[row]
                    arrseqs[row, :len(seqsrow)] = list(seqsrow)
                amask = np.char.islower(arrseqs)
                save_alleles = np.any(amask)

                ## send align1 to the bash shell
                ## TODO: check for pipe-overflow here and use files for i/o
                cmd1 = "echo -e '{}' | {} -quiet -in - ; echo {}"\
                       .format(cl1, ipyrad.bins.muscle, "//")
                print(cmd1, file=proc.stdin)

                ## read the stdout by line until splitter is reached
                for line in iter(proc.stdout.readline, "//\n"):
                    align1 += line

                ## send align2 to the bash shell
                ## TODO: check for pipe-overflow here and use files for i/o
                cmd2 = "echo -e '{}' | {} -quiet -in - ; echo {}"\
                       .format(cl2, ipyrad.bins.muscle, "//")
                print(cmd2, file=proc.stdin)

                ## read the stdout by line until splitter is reached
                for line in iter(proc.stdout.readline, "//\n"):
                    align2 += line

                ## join the aligned read1 and read2 and ensure name order match
                la1 = align1[1:].split("\n>")
                la2 = align2[1:].split("\n>")
                dalign1 = dict([i.split("\n", 1) for i in la1])
                dalign2 = dict([i.split("\n", 1) for i in la2])
                keys = sorted(dalign1.keys(), key=DEREP)
                keys2 = sorted(dalign2.keys(), key=DEREP)

                ## Make sure R1 and R2 actually exist for each sample. If not
                ## bail out of this cluster.
                if not len(keys) == len(keys2):
                    LOGGER.error("R1 and R2 results differ in length: "\
                                    + "\nR1 - {}\nR2 - {}".format(keys, keys2))
                    continue

                ## impute allele (lowercase) info back into alignments
                for kidx, key in enumerate(keys):
                    concatseq = dalign1[key].replace("\n", "")+\
                                "nnnn"+dalign2[key].replace("\n", "")

                    ## impute alleles
                    if save_alleles:
                        newmask = np.zeros(len(concatseq), dtype=np.bool_)
                        ## check for indels and impute to amask
                        indidx = np.where(np.array(list(concatseq)) == "-")[0]
                        if indidx.size:
                            allrows = np.arange(amask.shape[1])
                            mask = np.ones(allrows.shape[0], dtype=np.bool_)
                            for idx in indidx:
                                if idx < mask.shape[0]:
                                    mask[idx] = False
                            not_idx = allrows[mask == 1]
                            ## fill in new data into all other spots
                            newmask[not_idx] = amask[kidx, :not_idx.shape[0]]
                        else:
                            newmask = amask[kidx]

                        ## lower the alleles
                        concatarr = np.array(list(concatseq))
                        concatarr[newmask] = np.char.lower(concatarr[newmask])
                        concatseq = concatarr.tostring()
                        #LOGGER.info(concatseq)

                    ## fill list with aligned data
                    aligned.append("{}\n{}".format(key, concatseq))

                ## put into a dict for writing to file
                #aligned = []
                #for key in keys:
                #    aligned.append("\n".join(
                #        [key,
                #         dalign1[key].replace("\n", "")+"nnnn"+\
                #         dalign2[key].replace("\n", "")]))
            except IndexError as inst:
                LOGGER.debug("Error in PE - ldx: {}".format())
                LOGGER.debug("Vars: {}".format(dict(globals(), **locals())))
                raise

            except ValueError:
                ## make back into strings
                cl1 = "\n".join(["\n".join(i) for i in zip(names, seqs)])

                ## store allele (lowercase) info
                shape = (len(seqs), max([len(i) for i in seqs]))
                arrseqs = np.zeros(shape, dtype="S1")
                for row in range(arrseqs.shape[0]):
                    seqsrow = seqs[row]
                    arrseqs[row, :len(seqsrow)] = list(seqsrow)
                amask = np.char.islower(arrseqs)
                save_alleles = np.any(amask)

                ## send align1 to the bash shell (TODO: check for pipe-overflow)
                cmd1 = "echo -e '{}' | {} -quiet -in - ; echo {}"\
                       .format(cl1, ipyrad.bins.muscle, "//")
                print(cmd1, file=proc.stdin)

                ## read the stdout by line until splitter is reached
                for line in iter(proc.stdout.readline, "//\n"):
                    align1 += line

                ## ensure name order match
                la1 = align1[1:].split("\n>")
                dalign1 = dict([i.split("\n", 1) for i in la1])
                keys = sorted(dalign1.keys(), key=DEREP)

                ## put into dict for writing to file
                for kidx, key in enumerate(keys):
                    concatseq = dalign1[key].replace("\n", "")
                    ## impute alleles
                    if save_alleles:
                        newmask = np.zeros(len(concatseq), dtype=np.bool_)
                        ## check for indels and impute to amask
                        indidx = np.where(np.array(list(concatseq)) == "-")[0]
                        if indidx.size:
                            allrows = np.arange(amask.shape[1])
                            mask = np.ones(allrows.shape[0], dtype=np.bool_)
                            for idx in indidx:
                                if idx < mask.shape[0]:
                                    mask[idx] = False
                            not_idx = allrows[mask == 1]
                            ## fill in new data into all other spots
                            newmask[not_idx] = amask[kidx, :not_idx.shape[0]]
                        else:
                            newmask = amask[kidx]

                        ## lower the alleles
                        concatarr = np.array(list(concatseq))
                        concatarr[newmask] = np.char.lower(concatarr[newmask])
                        concatseq = concatarr.tostring()

                    ## fill list with aligned data
                    aligned.append("{}\n{}".format(key, concatseq))
                ## put aligned locus in list
                #aligned.append("\n".join(inner_aligned))

            ## enforce maxlen on aligned seqs
            aseqs = np.vstack([list(i.split("\n")[1]) for i in aligned])
            LOGGER.info("\naseqs here: %s", aseqs)

            ## index names by snames order
            sidxs = [snames.index(key.rsplit("_", 1)[0]) for key in keys]
            thislen = min(maxlen, aseqs.shape[1])
            for idx in xrange(aseqs.shape[0]):
                ## enter into stack
                newn = aligned[idx].split(";", 1)[0]
                #newn = key[idx].split(";", 1)[0]
                istack.append("{}\n{}".format(newn, aseqs[idx, :thislen].tostring()))
                ## name index in sorted list (indels order)
                sidx = sidxs[idx]
                indels[sidx, ldx, :thislen] = aseqs[idx, :thislen] == "-"

        if istack:
            allstack.append("\n".join(istack))
            #LOGGER.debug("\n\nSTACK (%s)\n%s\n", duples[ldx], "\n".join(istack))

    ## cleanup
    proc.stdout.close()
    if proc.stderr:
        proc.stderr.close()
    proc.stdin.close()
    proc.wait()

    #LOGGER.info("\n\nALLSTACK %s\n", "\n".join(i) for i in allstack[:5]])

    ## write to file after
    odx = chunk.rsplit("_")[-1]
    alignfile = os.path.join(data.tmpdir, "align_{}.fa".format(odx))
    with open(alignfile, 'wb') as outfile:
        outfile.write("\n//\n//\n".join(allstack)+"\n")
        os.remove(chunk)

    ## save indels array to tmp dir
    ifile = os.path.join(data.tmpdir, "indels_{}.tmp.npy".format(odx))
    np.save(ifile, indels)
    dfile = os.path.join(data.tmpdir, "duples_{}.tmp.npy".format(odx))
    np.save(dfile, duples)

Response: aligns the samples in chunk with the next cluster


Instruction:
Explain what the following Python 3 code does
def multi_muscle_align(data, samples, ipyclient):
    LOGGER.info("starting alignments")

    ## get client
    lbview = ipyclient.load_balanced_view()
    start = time.time()
    printstr = " aligning clusters     | {} | s6 |"
    elapsed = datetime.timedelta(seconds=int(time.time()-start))
    progressbar(20, 0, printstr.format(elapsed), spacer=data._spacer)

    ## submit clustbits as jobs to engines. The chunkfiles are removed when they
    ## are finished so this job can even be restarted if it was half finished,
    ## though that is probably rare.
    path = os.path.join(data.tmpdir, data.name + ".chunk_*")
    clustbits = glob.glob(path)
    jobs = {}
    for idx in xrange(len(clustbits)):
        args = [data, samples, clustbits[idx]]
        jobs[idx] = lbview.apply(persistent_popen_align3, *args)
    allwait = len(jobs)
    elapsed = datetime.timedelta(seconds=int(time.time()-start))
    progressbar(20, 0, printstr.format(elapsed), spacer=data._spacer)

    ## print progress while bits are aligning
    while 1:
        finished = [i.ready() for i in jobs.values()]
        fwait = sum(finished)
        elapsed = datetime.timedelta(seconds=int(time.time()-start))
        progressbar(allwait, fwait, printstr.format(elapsed), spacer=data._spacer)
        time.sleep(0.1)
        if all(finished):
            break

    ## check for errors in muscle_align_across
    keys = jobs.keys()
    for idx in keys:
        if not jobs[idx].successful():
            LOGGER.error("error in persistent_popen_align %s", jobs[idx].exception())
            raise IPyradWarningExit("error in step 6 {}".format(jobs[idx].exception()))
        del jobs[idx]
    print("")

Response: This function is used to align the cluster bits with muscle.


Instruction:
Make a summary of the following Python 3 code
def concatclusts(outhandle, alignbits):
    with gzip.open(outhandle, 'wb') as out:
        for fname in alignbits:
            with open(fname) as infile:
                out.write(infile.read()+"//\n//\n")

Response: concatenates sorted aligned cluster tmpfiles removes them


Instruction:
How would you code a function in Python 3 to
build the indels array and catclust. gz file from the aligned clusters.

Response:
def build_indels(data, samples, ipyclient):
    """
    Builds the indels array and catclust.gz file from the aligned clusters.
    Building catclust is very fast. Entering indels into h5 array is a bit
    slow but can probably be sped up. (todo). NOT currently parallelized.
    """

    ## progress bars
    lbview = ipyclient.load_balanced_view()
    start = time.time()
    printstr = " database indels       | {} | s6 |"
    njobs = len(glob.glob(os.path.join(data.tmpdir, "align_*.fa"))) + 1

    ## build tmparrs
    async = lbview.apply(build_tmp_h5, *(data, samples))

    ## track progress
    while 1:
        elapsed = datetime.timedelta(seconds=int(time.time()-start))
        ready = bool(async.ready())
        progressbar(njobs, ready, printstr.format(elapsed), spacer=data._spacer)
        if ready:
            break
        else:
            time.sleep(0.1)

    ## check for errors
    if not async.successful():
        raise IPyradWarningExit(async.result())

    ## start subfunc
    async = lbview.apply(sub_build_indels, *(data, samples))

    prog = 1
    while 1:
        elapsed = datetime.timedelta(seconds=int(time.time()-start))
        if async.stdout:
            prog = int(async.stdout.split()[-1])+1
        progressbar(njobs, prog, printstr.format(elapsed), spacer=data._spacer)
        if async.ready():
            break
        else:
            time.sleep(0.1)

    ## check for errors
    if not async.successful():
        raise IPyradWarningExit(async.result())
    print("")

    ## prepare for next substep by removing the singlecat result files if
    ## they exist.
    snames = [i.name for i in samples]
    snames.sort()
    smpios = [os.path.join(data.dirs.across, i+'.tmp.h5') for i in snames]
    for smpio in smpios:
        if os.path.exists(smpio):
            os.remove(smpio)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def sub_build_indels(data, samples):

    ## get file handles
    indelfiles = glob.glob(os.path.join(data.tmpdir, "indels_*.tmp.npy"))
    alignbits = glob.glob(os.path.join(data.tmpdir, "align_*.fa"))

    ## sort into input order by chunk names
    indelfiles.sort(key=lambda x: int(x.rsplit("_", 1)[-1][:-8]))
    alignbits.sort(key=lambda x: int(x.rsplit("_", 1)[-1][:-3]))
    LOGGER.info("indelfiles %s", indelfiles)
    LOGGER.info("alignbits %s", alignbits)
    chunksize = int(indelfiles[0].rsplit("_", 1)[-1][:-8])

    ## concatenate finished seq clusters into a tmp file
    outhandle = os.path.join(data.dirs.across, data.name+"_catclust.gz")
    concatclusts(outhandle, alignbits)

    ## get dims for full indel array
    maxlen = data._hackersonly["max_fragment_length"] + 20
    nloci = get_nloci(data)
    LOGGER.info("maxlen inside build is %s", maxlen)
    LOGGER.info("nloci for indels %s", nloci)

    ## INIT TEMP INDEL ARRAY
    ## build an indel array for ALL loci in cat.clust.gz,
    ## chunked so that individual samples can be pulled out
    ipath = os.path.join(data.dirs.across, data.name+".tmp.indels.hdf5")
    with h5py.File(ipath, 'w') as io5:
        iset = io5.create_dataset(
            "indels",
            shape=(len(samples), nloci, maxlen),
            dtype=np.bool_,
            chunks=(1, chunksize, maxlen))

        ## again make sure names are ordered right
        samples.sort(key=lambda x: x.name)

        #iset.attrs["chunksize"] = (1, data.nloci, maxlen)
        iset.attrs["samples"] = [i.name for i in samples]

        ## enter all tmpindel arrays into full indel array
        done = 0
        init = 0
        for indf in indelfiles:
            end = int(indf.rsplit("_", 1)[-1][:-8])
            inarr = np.load(indf)
            LOGGER.info('inarr shape %s', inarr.shape)
            LOGGER.info('iset shape %s', iset.shape)
            iset[:, init:end, :] = inarr[:, :end-init]
            init += end-init
            done += 1
            print(done)

Response: sub func in build_indels


Instruction:
Write a Python 3 function that can
distribute 'cluster()' function to an ipyclient to make sure it runs on a high memory node.

Response:
def call_cluster(data, noreverse, ipyclient):
    """
    distributes 'cluster()' function to an ipyclient to make sure it runs
    on a high memory node.
    """
    ## Find host with the most engines, for now just using first.
    lbview = ipyclient.load_balanced_view()

    ## request engine data, skips busy engines.
    asyncs = {}
    for eid in ipyclient.ids:
        engine = ipyclient[eid]
        if not engine.outstanding:
            asyncs[eid] = engine.apply(socket.gethostname)
    ## get results
    hosts = {}
    for key in asyncs:
        hosts[key] = asyncs[key].get()
    ## count them
    results = {}
    for eid, hostname in hosts.items():
        if hostname in results:
            results[hostname].append(eid)
        else:
            results[hostname] = [eid]

    ## which is largest
    hosts = sorted(results.items(), key=lambda x: len(x[1]), reverse=True)
    _, eids = hosts[0]
    bighost = ipyclient[eids[0]]

    ## nthreads is len eids, or ipcluster.threads, unless ipcluster.threads
    ## is really small, then we assume threads should not apply here.
    ##    ipyrad -p params.txt -s 6 -c 20 would give:
    ##    min(20, max(2, 10)) = 8
    ## while
    ##    ipyrad -p params.txt -s 6 -c 20 -t 4 would give:
    ##    min(20, max(4, 10)) = 10
    ## and
    ##    ipyrad -p params.txt -s 6 -c 20 -t 15 would give:
    ##    min(20, max(15, 10)) = 15
    ## and
    ##    ipyrad -p params.txt -s 6 -c 16 --MPI (on 2 X 8-core nodes) would give:
    ##    min(8, max(2, 10)) = 8
    nthreads = min(len(eids), max(data._ipcluster["threads"], 10))

    ## submit job to the host with the most
    async = bighost.apply(cluster, *(data, noreverse, nthreads))
    #async = lbview.apply(cluster, *(data, noreverse, nthreads))

    ## track progress
    prog = 0
    start = time.time()
    printstr = " clustering across     | {} | s6 |"

    while 1:
        if async.stdout:
            prog = int(async.stdout.split()[-1])
        elapsed = datetime.timedelta(seconds=int(time.time() - start))
        progressbar(100, prog, printstr.format(elapsed), spacer=data._spacer)
        if async.ready():
            progressbar(100, prog, printstr.format(elapsed), spacer=data._spacer)
            print("")
            break
        else:
            time.sleep(0.5)

    ## store log result
    ipyclient.wait()
    data.stats_files.s6 = os.path.join(data.dirs.across, "s6_cluster_stats.txt")


Instruction:
Can you generate a brief explanation for the following Python 3 code
def cluster(data, noreverse, nthreads):

    ## input and output file handles
    cathaplos = os.path.join(data.dirs.across, data.name+"_catshuf.tmp")
    uhaplos = os.path.join(data.dirs.across, data.name+".utemp")
    hhaplos = os.path.join(data.dirs.across, data.name+".htemp")
    logfile = os.path.join(data.dirs.across, "s6_cluster_stats.txt")

    ## parameters that vary by datatype
    ## (too low of cov values yield too many poor alignments)
    strand = "plus"
    cov = 0.75    ##0.90
    if data.paramsdict["datatype"] in ["gbs", "2brad"]:
        strand = "both"
        cov = 0.60
    elif data.paramsdict["datatype"] == "pairgbs":
        strand = "both"
        cov = 0.75   ##0.90

    ## nthreads is calculated in 'call_cluster()'
    cmd = [ipyrad.bins.vsearch,
           "-cluster_smallmem", cathaplos,
           "-strand", strand,
           "-query_cov", str(cov),
           "-minsl", str(0.5),
           "-id", str(data.paramsdict["clust_threshold"]),
           "-userout", uhaplos,
           "-notmatched", hhaplos,
           "-userfields", "query+target+qstrand",
           "-maxaccepts", "1",
           "-maxrejects", "0",
           "-fasta_width", "0",
           "-threads", str(nthreads), #"0",
           "-fulldp",
           "-usersort",
           "-log", logfile]

    ## override reverse clustering option
    if noreverse:
        strand = "plus"  # -leftjust "

    try:
        ## this seems to start vsearch on a different pid than the engine
        ## and so it's hard to kill...
        LOGGER.info(cmd)
        (dog, owner) = pty.openpty()
        proc = sps.Popen(cmd, stdout=owner, stderr=owner, close_fds=True)

        prog = 0
        newprog = 0
        while 1:
            isdat = select.select([dog], [], [], 0)
            if isdat[0]:
                dat = os.read(dog, 80192)
            else:
                dat = ""
            if "Clustering" in dat:
                try:
                    newprog = int(dat.split()[-1][:-1])
                ## may raise value error when it gets to the end
                except ValueError:
                    pass

            ## break if done
            ## catches end chunk of printing if clustering went really fast
            elif "Clusters:" in dat:
                LOGGER.info("ended vsearch tracking loop")
                break
            else:
                time.sleep(0.1)
            ## print progress
            if newprog != prog:
                print(newprog)
                prog = newprog

        ## another catcher to let vsearch cleanup after clustering is done
        proc.wait()
        print(100)

    except KeyboardInterrupt:
        LOGGER.info("interrupted vsearch here: %s", proc.pid)
        os.kill(proc.pid, 2)
        raise KeyboardInterrupt()
    except sps.CalledProcessError as inst:
        raise IPyradWarningExit("""
        Error in vsearch: \n{}\n{}""".format(inst, sps.STDOUT))
    except OSError as inst:
        raise IPyradWarningExit("""
        Failed to allocate pty: \n{}""".format(inst))

    finally:
        data.stats_files.s6 = logfile

Response: Calls vsearch for clustering across samples.


Instruction:
Make a summary of the following Python 3 code
def build_h5_array(data, samples, nloci):

    ## sort to ensure samples will be in alphabetical order, tho they should be.
    samples.sort(key=lambda x: x.name)

    ## get maxlen dim
    maxlen = data._hackersonly["max_fragment_length"] + 20
    LOGGER.info("maxlen inside build_h5_array is %s", maxlen)
    LOGGER.info("nloci inside build_h5_array is %s", nloci)

    ## open new h5 handle
    data.clust_database = os.path.join(data.dirs.across, data.name+".clust.hdf5")
    io5 = h5py.File(data.clust_database, 'w')

    ## chunk to approximately 2 chunks per core
    chunks = ((nloci // (data.cpus*2)) + (nloci % (data.cpus*2)))

    ## Number of elements in hdf5 chunk may not exceed 500MB
    ## This is probably not actually optimal, to have such
    ## enormous chunk sizes, could probably explore efficiency
    ## of smaller chunk sizes on very very large datasets
    chunklen = chunks * len(samples) * maxlen * 4
    while chunklen > int(500e6):
        chunks = (chunks // 2) + (chunks % 2)
        chunklen = chunks * len(samples) * maxlen * 4
    LOGGER.info("chunks in build_h5_array: %s", chunks)

    data.chunks = chunks
    LOGGER.info("nloci is %s", nloci)
    LOGGER.info("chunks is %s", data.chunks)

    ## INIT FULL CATG ARRAY
    ## store catgs with a .10 loci chunk size
    supercatg = io5.create_dataset("catgs", (nloci, len(samples), maxlen, 4),
                                    dtype=np.uint32,
                                    chunks=(chunks, 1, maxlen, 4),
                                    compression="gzip")
    superseqs = io5.create_dataset("seqs", (nloci, len(samples), maxlen),
                                    dtype="|S1",
                                    #dtype=np.uint8,
                                    chunks=(chunks, len(samples), maxlen),
                                    compression='gzip')
    superalls = io5.create_dataset("nalleles", (nloci, len(samples)),
                                    dtype=np.uint8,
                                    chunks=(chunks, len(samples)),
                                    compression="gzip")
    superchroms = io5.create_dataset("chroms", (nloci, 3),
                                     dtype=np.int64,
                                     chunks=(chunks, 3),
                                     compression="gzip")

    ## allele count storage
    supercatg.attrs["chunksize"] = (chunks, 1, maxlen, 4)
    supercatg.attrs["samples"] = [i.name for i in samples]
    superseqs.attrs["chunksize"] = (chunks, len(samples), maxlen)
    superseqs.attrs["samples"] = [i.name for i in samples]
    superalls.attrs["chunksize"] = (chunks, len(samples))
    superalls.attrs["samples"] = [i.name for i in samples]
    superchroms.attrs["chunksize"] = (chunks, len(samples))
    superchroms.attrs["samples"] = [i.name for i in samples]

    ## array for pair splits locations, dup and ind filters
    io5.create_dataset("splits", (nloci, ), dtype=np.uint16)
    io5.create_dataset("duplicates", (nloci, ), dtype=np.bool_)

    ## close the big boy
    io5.close()

Response: Build a new h5 array for pre - filtered loci.


Instruction:
Here you have a function in Python 3, explain what it does
def fill_dups_arr(data):
    ## build the duplicates array
    duplefiles = glob.glob(os.path.join(data.tmpdir, "duples_*.tmp.npy"))
    duplefiles.sort(key=lambda x: int(x.rsplit("_", 1)[-1][:-8]))

    ## enter the duplicates filter into super h5 array
    io5 = h5py.File(data.clust_database, 'r+')
    dfilter = io5["duplicates"]

    ## enter all duple arrays into full duplicates array
    init = 0
    for dupf in duplefiles:
        end = int(dupf.rsplit("_", 1)[-1][:-8])
        inarr = np.load(dupf)
        dfilter[init:end] = inarr
        init += end-init
        #os.remove(dupf)
    #del inarr

    ## continued progress bar
    LOGGER.info("all duplicates: %s", dfilter[:].sum())
    io5.close()

Response: fills the duplicates array from the multi_muscle_align tmp files


Instruction:
Make a summary of the following Python 3 code
def build_tmp_h5(data, samples):
    ## get samples and names, sorted
    snames = [i.name for i in samples]
    snames.sort()

    ## Build an array for quickly indexing consens reads from catg files.
    ## save as a npy int binary file.
    uhandle = os.path.join(data.dirs.across, data.name+".utemp.sort")
    bseeds = os.path.join(data.dirs.across, data.name+".tmparrs.h5")

    ## send as first async1 job
    get_seeds_and_hits(uhandle, bseeds, snames)

Response: build tmp h5 arrays that can return quick access for nloci


Instruction:
Here you have a function in Python 3, explain what it does
def get_nloci(data):
    bseeds = os.path.join(data.dirs.across, data.name+".tmparrs.h5")
    with h5py.File(bseeds) as io5:
        return io5["seedsarr"].shape[0]

Response: return nloci from the tmp h5 arr


Instruction:
How would you code a function in Python 3 to
build a array of ints from the utemp. sort file and creates a array of ints from the utemp. sort file.

Response:
def get_seeds_and_hits(uhandle, bseeds, snames):
    """
    builds a seeds and hits (uarr) array of ints from the utemp.sort file.
    Saves outputs to files ...
    """
    ## Get max name length. Allow for trailing _ + up to 9 digits
    ## of numbers of loci (an astronomical number of unique loci)
    maxlen_names = np.max(map(len, snames)) + 10
    ## read in the utemp.sort file
    updf = np.loadtxt(uhandle, dtype="S".format(maxlen_names))

    ## Get seeds for all matches from usort
    seeds = np.unique(updf[:, 1])
    seedsarr = np.column_stack([
                    np.arange(len(seeds)),
                   [i.rsplit("_", 1)[0] for i in seeds],
                   [i.rsplit("_", 1)[1] for i in seeds]])
    seedsarr[:, 1] = [snames.index(i) for i in seedsarr[:, 1]]
    seedsarr = seedsarr.astype(np.int64)
    LOGGER.info("got a seedsarr %s", seedsarr.shape)

    ## Get matches from usort and create an array for fast entry
    uarr = np.zeros((updf.shape[0], 3), dtype=np.int64)
    idx = -1
    lastloc = None
    for ldx in xrange(updf.shape[0]):
        tloc = updf[ldx, 1]
        if tloc != lastloc:
            idx += 1
        uarr[ldx, 0] = idx
        lastloc = tloc
    ## create a column with sample index
    uarr[:, 1] = [int(snames.index(i.rsplit("_", 1)[0])) for i in updf[:, 0]]
    ## create a column with only consens index for sample
    uarr[:, 2] = [int(i.rsplit("_", 1)[1]) for i in updf[:, 0]]
    uarr = uarr.astype(np.int64)
    LOGGER.info("got a uarr %s", uarr.shape)

    ## save as h5 to we can grab by sample slices
    with h5py.File(bseeds, 'w') as io5:
        io5.create_dataset("seedsarr", data=seedsarr, dtype=np.int64)
        io5.create_dataset("uarr", data=uarr, dtype=np.int64)


Instruction:
How would you code a function in Python 3 to
create a new multicat dataset.

Response:
def new_multicat(data, samples, ipyclient):
    """
    Calls 'singlecat()' for all samples to build index files.
    """

    ## track progress
    LOGGER.info("in the multicat")
    start = time.time()
    printstr = " indexing clusters     | {} | s6 |"

    ## Build the large h5 array. This will write a new HDF5 file and overwrite
    ## existing data.
    nloci = get_nloci(data)
    build_h5_array(data, samples, nloci)

    ## parallel client (reserve engine 0 for data entry), if/else here in case
    ## user has only one engine.
    if len(ipyclient) > 1:
        filler = ipyclient.load_balanced_view(targets=[0])
        smallview = ipyclient.load_balanced_view(targets=ipyclient.ids[1::2])
    else:
        filler = ipyclient.load_balanced_view(targets=[0])
        smallview = ipyclient.load_balanced_view(targets=[0])

    ## First submit a sleeper job as temp_flag for cleanups
    last_sample = 0
    cleanups = {}
    cleanups[last_sample] = filler.apply(time.sleep, 0.0)

    ## fill the duplicates filter array
    async = smallview.apply(fill_dups_arr, data)
    while 1:
        elapsed = datetime.timedelta(seconds=int(time.time() - start))
        progressbar(20, 0, printstr.format(elapsed), spacer=data._spacer)
        time.sleep(0.1)
        if async.ready():
            break
    if not async.successful():
        raise IPyradWarningExit(async.result())

    ## Get all existing .tmp.h5 files. If files exist then assume that we are
    ## restarting an interrupted job. We need to check for each one whether it
    ## has it finished being built, and whether it has been written to the
    ## large array yet.
    snames = [i.name for i in samples]
    snames.sort()
    smpios = {i:os.path.join(data.dirs.across, i+'.tmp.h5') for i in snames}

    ## send 'singlecat()' jobs to engines
    bseeds = os.path.join(data.dirs.across, data.name+".tmparrs.h5")
    jobs = {}
    for sample in samples:
        sidx = snames.index(sample.name)
        args = (data, sample, bseeds, sidx, nloci)

        ## Only build it if it doesn't already exist. Singlecat removes
        ## unfinished files if interrupted, so .tmp.h5 should not exist
        ## unless the file is ready to be entered.
        if not os.path.exists(smpios[sample.name]):
            jobs[sample.name] = smallview.apply(singlecat, *args)

    ## track progress of singlecat jobs and submit writing jobs for finished
    ## singlecat files (.tmp.h5).
    alljobs = len(jobs)
    while 1:
        ## check for finished jobs
        curkeys = jobs.keys()
        for key in curkeys:
            async = jobs[key]
            if async.ready():
                if async.successful():
                    ## submit cleanup for finished job
                    args = (data, data.samples[key], snames.index(key))
                    with filler.temp_flags(after=cleanups[last_sample]):
                        cleanups[key] = filler.apply(write_to_fullarr, *args)
                        last_sample = key
                        del jobs[key]
                else:
                    if not async.successful():
                        raise IPyradWarningExit(async.result())

        ## print progress or break
        elapsed = datetime.timedelta(seconds=int(time.time() - start))
        progressbar(alljobs, alljobs-len(jobs), printstr.format(elapsed), spacer=data._spacer)
        time.sleep(0.1)
        if not jobs:
            break

    ## add the dask_chroms func for reference data
    if 'reference' in data.paramsdict["assembly_method"]:
        with filler.temp_flags(after=cleanups.values()):
            cleanups['ref'] = filler.apply(dask_chroms, *(data, samples))

    ## ------- print breakline between indexing and writing database ---------
    print("")

    ## track progress of databaseing
    start = time.time()
    printstr = " building database     | {} | s6 |"
    while 1:
        finished = [i for i in cleanups.values() if i.ready()]
        elapsed = datetime.timedelta(seconds=int(time.time() - start))
        progressbar(len(cleanups), len(finished), printstr.format(elapsed), spacer=data._spacer)
        time.sleep(0.1)
        ## break if one failed, or if finished
        if not all([i.successful() for i in finished]):
            break
        if len(cleanups) == len(finished):
            break

    ## check for errors
    for job in cleanups:
        if cleanups[job].ready():
            if not cleanups[job].successful():
                raise IPyradWarningExit((job, cleanups[job].result()))


Instruction:
Write a Python 3 function that can
run a singlecat and cleanup jobs for each sample.

Response:
def multicat(data, samples, ipyclient):
    """
    Runs singlecat and cleanup jobs for each sample.
    For each sample this fills its own hdf5 array with catg data & indels.
    This is messy, could use simplifiying.
    """

    ## progress ticker
    start = time.time()
    printstr = " indexing clusters     | {} | s6 |"
    elapsed = datetime.timedelta(seconds=int(time.time() - start))
    progressbar(20, 0, printstr.format(elapsed))

    ## parallel client
    lbview = ipyclient.load_balanced_view()

    ## First submit a sleeper job as temp_flag for cleanups
    last_sample = 0
    cleanups = {}
    cleanups[last_sample] = lbview.apply(time.sleep, 0.0)

    ## get samples and names, sorted
    snames = [i.name for i in samples]
    snames.sort()

    ## Build an array for quickly indexing consens reads from catg files.
    ## save as a npy int binary file.
    uhandle = os.path.join(data.dirs.across, data.name+".utemp.sort")
    bseeds = os.path.join(data.dirs.across, data.name+".tmparrs.h5")

    ## send as first async1 job
    async1 = lbview.apply(get_seeds_and_hits, *(uhandle, bseeds, snames))
    async2 = lbview.apply(fill_dups_arr, data)

    ## progress bar for seed/hit sorting
    while not (async1.ready() and async2.ready()):
        elapsed = datetime.timedelta(seconds=int(time.time() - start))
        progressbar(20, 0, printstr.format(elapsed))
        time.sleep(0.1)
    if not async1.successful():
        raise IPyradWarningExit("error in get_seeds: %s", async1.exception())
    if not async2.successful():
        raise IPyradWarningExit("error in fill_dups: %s", async2.exception())

    ## make a limited njobs view based on mem limits
    ## is using smallview necessary? (yes, it is for bad libraries)
    smallview = ipyclient.load_balanced_view(targets=ipyclient.ids[::2])

    ## make sure there are no old tmp.h5 files
    smpios = [os.path.join(data.dirs.across, sample.name+'.tmp.h5') \
              for sample in samples]
    for smpio in smpios:
        if os.path.exists(smpio):
            os.remove(smpio)

    ## send 'singlecat()' jobs to engines
    jobs = {}
    for sample in samples:
        sidx = snames.index(sample.name)
        jobs[sample.name] = smallview.apply(singlecat, *(data, sample, bseeds, sidx))

    ## check for finished and submit disk-writing job when finished
    alljobs = len(jobs)
    while 1:
        ## check for finished jobs
        curkeys = jobs.keys()
        for key in curkeys:
            async = jobs[key]
            if async.ready():
                if async.successful():
                    ## submit cleanup for finished job
                    args = (data, data.samples[key], snames.index(key))
                    with lbview.temp_flags(after=cleanups[last_sample]):
                        cleanups[key] = lbview.apply(write_to_fullarr, *args)
                    last_sample = key
                    del jobs[key]
                else:
                    err = jobs[key].exception()
                    errmsg = "singlecat error: {} {}".format(key, err)
                    raise IPyradWarningExit(errmsg)
        ## print progress or break
        elapsed = datetime.timedelta(seconds=int(time.time() - start))
        progressbar(alljobs, alljobs-len(jobs), printstr.format(elapsed))
        time.sleep(0.1)
        if not jobs:
            break

    ## add the dask_chroms func for reference data
    if 'reference' in data.paramsdict["assembly_method"]:
        with lbview.temp_flags(after=cleanups.values()):
            cleanups['ref'] = lbview.apply(dask_chroms, *(data, samples))

    ## wait for "write_to_fullarr" jobs to finish
    print("")
    start = time.time()
    printstr = " building database     | {} | s6 |"
    while 1:
        finished = [i for i in cleanups.values() if i.ready()]
        elapsed = datetime.timedelta(seconds=int(time.time() - start))
        progressbar(len(cleanups), len(finished), printstr.format(elapsed))
        time.sleep(0.1)
        ## break if one failed, or if finished
        if not all([i.successful() for i in finished]):
            break
        if len(cleanups) == len(finished):
            break

    ## check for errors
    for job in cleanups:
        if cleanups[job].ready():
            if not cleanups[job].successful():
                err = " error in write_to_fullarr ({}) {}"\
                     .format(job, cleanups[job].result())
                LOGGER.error(err)
                raise IPyradWarningExit(err)

    ## remove large indels array file and singlecat tmparr file
    ifile = os.path.join(data.dirs.across, data.name+".tmp.indels.hdf5")
    if os.path.exists(ifile):
        os.remove(ifile)
    if os.path.exists(bseeds):
        os.remove(bseeds)
    for sh5 in [os.path.join(data.dirs.across, i.name+".tmp.h5") for i in samples]:
        os.remove(sh5)

    ## print final progress
    elapsed = datetime.timedelta(seconds=int(time.time() - start))
    progressbar(10, 10, printstr.format(elapsed))
    print("")


Instruction:
Can you generate the documentation for the following Python 3 function
def singlecat(data, sample, bseeds, sidx, nloci):

    LOGGER.info("in single cat here")
    ## enter ref data?
    isref = 'reference' in data.paramsdict["assembly_method"]

    ## grab seeds and hits info for this sample
    with h5py.File(bseeds, 'r') as io5:
        ## get hits just for this sample and sort them by sample order index
        hits = io5["uarr"][:]
        hits = hits[hits[:, 1] == sidx, :]
        #hits = hits[hits[:, 2].argsort()]
        ## get seeds just for this sample and sort them by sample order index
        seeds = io5["seedsarr"][:]
        seeds = seeds[seeds[:, 1] == sidx, :]
        #seeds = seeds[seeds[:, 2].argsort()]
        full = np.concatenate((seeds, hits))
        full = full[full[:, 0].argsort()]

    ## still using max+20 len limit, rare longer merged reads get trimmed
    ## we need to allow room for indels to be added too
    maxlen = data._hackersonly["max_fragment_length"] + 20

    ## we'll fill a new catg and alleles arr for this sample in locus order,
    ## which is known from seeds and hits
    ocatg = np.zeros((nloci, maxlen, 4), dtype=np.uint32)
    onall = np.zeros(nloci, dtype=np.uint8)
    ochrom = np.zeros((nloci, 3), dtype=np.int64)

    ## grab the sample's data and write to ocatg and onall
    if not sample.files.database:
        raise IPyradWarningExit("missing catg file - {}".format(sample.name))

    with h5py.File(sample.files.database, 'r') as io5:
        ## get it and delete it
        catarr = io5["catg"][:]
        tmp = catarr[full[:, 2], :maxlen, :]
        del catarr
        ocatg[full[:, 0], :tmp.shape[1], :] = tmp
        del tmp

        ## get it and delete it
        nall = io5["nalleles"][:]
        onall[full[:, 0]] = nall[full[:, 2]]
        del nall

        ## fill the reference data
        if isref:
            chrom = io5["chroms"][:]
            ochrom[full[:, 0]] = chrom[full[:, 2]]
            del chrom

    ## get indel locations for this sample
    ipath = os.path.join(data.dirs.across, data.name+".tmp.indels.hdf5")
    with h5py.File(ipath, 'r') as ih5:
        indels = ih5["indels"][sidx, :, :maxlen]

    ## insert indels into ocatg
    newcatg = inserted_indels(indels, ocatg)
    del ocatg, indels

    ## save individual tmp h5 data
    smpio = os.path.join(data.dirs.across, sample.name+'.tmp.h5')
    with h5py.File(smpio, 'w') as oh5:
        oh5.create_dataset("icatg", data=newcatg, dtype=np.uint32)
        oh5.create_dataset("inall", data=onall, dtype=np.uint8)
        if isref:
            oh5.create_dataset("ichrom", data=ochrom, dtype=np.int64)

Response: Create a single segment for each sample in the locus order.


Instruction:
Write a Python 3 script for
writing arrays to h5 disk

Response:
def write_to_fullarr(data, sample, sidx):
    """ writes arrays to h5 disk """

    ## enter ref data?
    #isref = 'reference' in data.paramsdict["assembly_method"]
    LOGGER.info("writing fullarr %s %s", sample.name, sidx)

    ## save big arrays to disk temporarily
    with h5py.File(data.clust_database, 'r+') as io5:
        ## open views into the arrays we plan to fill
        chunk = io5["catgs"].attrs["chunksize"][0]
        catg = io5["catgs"]
        nall = io5["nalleles"]

        ## adding an axis to newcatg makes it write about 1000X faster.
        smpio = os.path.join(data.dirs.across, sample.name+'.tmp.h5')
        with h5py.File(smpio) as indat:

            ## grab all of the data from this sample's arrays
            newcatg = indat["icatg"] #[:]
            onall = indat["inall"]   #[:]

            ## enter it into the full array one chunk at a time
            for cidx in xrange(0, catg.shape[0], chunk):
                end = cidx + chunk
                catg[cidx:end, sidx:sidx+1, :] = np.expand_dims(newcatg[cidx:end, :], axis=1)
                nall[:, sidx:sidx+1] = np.expand_dims(onall, axis=1)


Instruction:
How would you explain what the following Python 3 function does
def dask_chroms(data, samples):

    ## example concatenating with dask
    h5s = [os.path.join(data.dirs.across, s.name+".tmp.h5") for s in samples]
    handles = [h5py.File(i) for i in h5s]
    dsets = [i['/ichrom'] for i in handles]
    arrays = [da.from_array(dset, chunks=(10000, 3)) for dset in dsets]
    stack = da.stack(arrays, axis=2)

    ## max chrom (should we check for variable hits? if so, things can get wonk)
    maxchrom = da.max(stack, axis=2)[:, 0]

    ## max pos
    maxpos = da.max(stack, axis=2)[:, 2]

    ## min pos
    mask = stack == 0
    stack[mask] = 9223372036854775807  ## max int64 value
    minpos = da.min(stack, axis=2)[:, 1]
    final = da.stack([maxchrom, minpos, maxpos], axis=1)
    final.to_hdf5(data.clust_database, "/chroms")

    ## close the h5 handles
    _ = [i.close() for i in handles]

Response: A dask relay function to fill chroms for all samples


Instruction:
Here you have a function in Python 3, explain what it does
def inserted_indels(indels, ocatg):
    ## return copy with indels inserted
    newcatg = np.zeros(ocatg.shape, dtype=np.uint32)

    ## iterate over loci and make extensions for indels
    for iloc in xrange(ocatg.shape[0]):
        ## get indels indices
        indidx = np.where(indels[iloc, :])[0]
        if np.any(indidx):
            ## which new (empty) rows will be added
            allrows = np.arange(ocatg.shape[1])
            mask = np.ones(allrows.shape[0], dtype=np.bool_)
            for idx in indidx:
                mask[idx] = False
            not_idx = allrows[mask == 1]

            ## fill in new data into all other spots
            newcatg[iloc][not_idx] = ocatg[iloc, :not_idx.shape[0]]
        else:
            newcatg[iloc] = ocatg[iloc]
    return newcatg

Response: inserts indels into the catg array


Instruction:
Can you write a function in Python 3 where it
fills the superseqs array with seq data from cat. clust. clust and fill the edges array with information about paired split locations.

Response:
def fill_superseqs(data, samples):
    """
    Fills the superseqs array with seq data from cat.clust
    and fill the edges array with information about paired split locations.
    """

    ## load super to get edges
    io5 = h5py.File(data.clust_database, 'r+')
    superseqs = io5["seqs"]
    splits = io5["splits"]

    ## samples are already sorted
    snames = [i.name for i in samples]
    LOGGER.info("snames %s", snames)

    ## get maxlen again
    maxlen = data._hackersonly["max_fragment_length"] + 20
    LOGGER.info("maxlen inside fill_superseqs is %s", maxlen)

    ## data has to be entered in blocks
    infile = os.path.join(data.dirs.across, data.name+"_catclust.gz")
    clusters = gzip.open(infile, 'r')
    pairdealer = itertools.izip(*[iter(clusters)]*2)

    ## iterate over clusters
    chunks = superseqs.attrs["chunksize"]
    chunksize = chunks[0]
    done = 0
    iloc = 0
    cloc = 0
    chunkseqs = np.zeros(chunks, dtype="|S1")
    chunkedge = np.zeros(chunksize, dtype=np.uint16)

    while 1:
        try:
            done, chunk = clustdealer(pairdealer, 1)
        except IndexError:
            raise IPyradWarningExit("clustfile formatting error in %s", chunk)

        ## if chunk is full put into superseqs and reset counter
        if cloc == chunksize:
            LOGGER.info("cloc chunk writing %s", cloc)
            superseqs[iloc-cloc:iloc] = chunkseqs
            splits[iloc-cloc:iloc] = chunkedge
            ## reset chunkseqs, chunkedge, cloc
            cloc = 0
            chunkseqs = np.zeros((chunksize, len(samples), maxlen), dtype="|S1")
            chunkedge = np.zeros((chunksize), dtype=np.uint16)

        ## get seq and split it
        if chunk:
            try:
                fill = np.zeros((len(samples), maxlen), dtype="|S1")
                fill.fill("N")
                piece = chunk[0].strip().split("\n")
                names = piece[0::2]
                seqs = np.array([list(i) for i in piece[1::2]])

                ## fill in the separator if it exists
                separator = np.where(np.all(seqs == 'n', axis=0))[0]
                if np.any(separator):
                    chunkedge[cloc] = separator.min()

                ## fill in the hits
                ## seqs will be (5,) IF the seqs are variable lengths, which
                ## can happen if it had duplicaes AND there were indels, and
                ## so the indels did not get aligned
                try:
                    shlen = seqs.shape[1]
                except IndexError as inst:
                    shlen = min([len(x) for x in seqs])

                for name, seq in zip(names, seqs):
                    sidx = snames.index(name.rsplit("_", 1)[0])
                    #fill[sidx, :shlen] = seq[:maxlen]
                    fill[sidx, :shlen] = seq[:shlen]

                ## PUT seqs INTO local ARRAY
                chunkseqs[cloc] = fill

            except Exception as inst:
                LOGGER.info(inst)
                LOGGER.info("\nfill: %s\nshlen %s\nmaxlen %s", fill.shape, shlen, maxlen)
                LOGGER.info("dupe chunk \n{}".format("\n".join(chunk)))

            ## increase counters if there was a chunk
            cloc += 1
            iloc += 1
        if done:
            break

    ## write final leftover chunk
    superseqs[iloc-cloc:,] = chunkseqs[:cloc]
    splits[iloc-cloc:] = chunkedge[:cloc]

    ## close super
    io5.close()
    clusters.close()

    ## edges is filled with splits for paired data.
    LOGGER.info("done filling superseqs")


Instruction:
Given the following Python 3 function, write the documentation
def count_seeds(usort):
    with open(usort, 'r') as insort:
        cmd1 = ["cut", "-f", "2"]
        cmd2 = ["uniq"]
        cmd3 = ["wc"]
        proc1 = sps.Popen(cmd1, stdin=insort, stdout=sps.PIPE, close_fds=True)
        proc2 = sps.Popen(cmd2, stdin=proc1.stdout, stdout=sps.PIPE, close_fds=True)
        proc3 = sps.Popen(cmd3, stdin=proc2.stdout, stdout=sps.PIPE, close_fds=True)
        res = proc3.communicate()
        nseeds = int(res[0].split()[0])
        proc1.stdout.close()
        proc2.stdout.close()
        proc3.stdout.close()
    return nseeds

Response: count the number of seeds in a given order


Instruction:
Can you generate a brief explanation for the following Python 3 code
def sort_seeds(uhandle, usort):
    cmd = ["sort", "-k", "2", uhandle, "-o", usort]
    proc = sps.Popen(cmd, close_fds=True)
    proc.communicate()

Response: sort seeds from cluster results


Instruction:
Given the following Python 3 function, write the documentation
def build_clustbits(data, ipyclient, force):

    ## If you run this step then we clear all tmp .fa and .indel.h5 files
    if os.path.exists(data.tmpdir):
        shutil.rmtree(data.tmpdir)
        os.mkdir(data.tmpdir)

    ## parallel client
    lbview = ipyclient.load_balanced_view()
    start = time.time()
    printstr = " building clusters     | {} | s6 |"
    elapsed = datetime.timedelta(seconds=int(time.time()-start))
    progressbar(3, 0, printstr.format(elapsed), spacer=data._spacer)

    uhandle = os.path.join(data.dirs.across, data.name+".utemp")
    usort = os.path.join(data.dirs.across, data.name+".utemp.sort")

    async1 = ""
    ## skip usorting if not force and already exists
    if not os.path.exists(usort) or force:

        ## send sort job to engines. Sorted seeds allows us to work through
        ## the utemp file one locus at a time instead of reading all into mem.
        LOGGER.info("building reads file -- loading utemp file into mem")
        async1 = lbview.apply(sort_seeds, *(uhandle, usort))
        while 1:
            elapsed = datetime.timedelta(seconds=int(time.time()-start))
            progressbar(3, 0, printstr.format(elapsed), spacer=data._spacer)
            if async1.ready():
                break
            else:
                time.sleep(0.1)

    ## send count seeds job to engines.
    async2 = lbview.apply(count_seeds, usort)
    while 1:
        elapsed = datetime.timedelta(seconds=int(time.time()-start))
        progressbar(3, 1, printstr.format(elapsed), spacer=data._spacer)
        if async2.ready():
            break
        else:
            time.sleep(0.1)

    ## wait for both to finish while printing progress timer
    nseeds = async2.result()

    ## send the clust bit building job to work and track progress
    async3 = lbview.apply(sub_build_clustbits, *(data, usort, nseeds))
    while 1:
        elapsed = datetime.timedelta(seconds=int(time.time()-start))
        progressbar(3, 2, printstr.format(elapsed), spacer=data._spacer)
        if async3.ready():
            break
        else:
            time.sleep(0.1)
    elapsed = datetime.timedelta(seconds=int(time.time()-start))
    progressbar(3, 3, printstr.format(elapsed), spacer=data._spacer)
    print("")

    ## check for errors
    for job in [async1, async2, async3]:
        try:
            if not job.successful():
                raise IPyradWarningExit(job.result())
        except AttributeError:
            ## If we skip usorting then async1 == "" so the call to
            ## successful() raises, but we can ignore it.
            pass

Response: Builds the cluster from the utemp and htemp files and writes them to chunked files for aligning in muscle.


Instruction:
Make a summary of the following Python 3 code
def sub_build_clustbits(data, usort, nseeds):

    ## load FULL concat fasta file into a dict. This could cause RAM issues.
    ## this file has iupac codes in it, not ambigs resolved, and is gzipped.
    LOGGER.info("loading full _catcons file into memory")
    allcons = {}
    conshandle = os.path.join(data.dirs.across, data.name+"_catcons.tmp")
    with gzip.open(conshandle, 'rb') as iocons:
        cons = itertools.izip(*[iter(iocons)]*2)
        for namestr, seq in cons:
            nnn, sss = [i.strip() for i in namestr, seq]
            allcons[nnn[1:]] = sss

    ## set optim to approximately 4 chunks per core. Smaller allows for a bit
    ## cleaner looking progress bar. 40 cores will make 160 files.
    optim = ((nseeds // (data.cpus*4)) + (nseeds % (data.cpus*4)))
    LOGGER.info("building clustbits, optim=%s, nseeds=%s, cpus=%s",
                optim, nseeds, data.cpus)

    ## iterate through usort grabbing seeds and matches
    with open(usort, 'rb') as insort:
        ## iterator, seed null, and seqlist null
        isort = iter(insort)
        loci = 0
        lastseed = 0
        fseqs = []
        seqlist = []
        seqsize = 0

        while 1:
            ## grab the next line
            try:
                hit, seed, ori = isort.next().strip().split()
            except StopIteration:
                break

            try:
                ## if same seed, append match
                if seed != lastseed:
                    ## store the last fseq, count it, and clear it
                    if fseqs:
                        seqlist.append("\n".join(fseqs))
                        seqsize += 1
                        fseqs = []
                    ## occasionally write to file
                    if seqsize >= optim:
                        if seqlist:
                            loci += seqsize
                            with open(os.path.join(data.tmpdir,
                                data.name+".chunk_{}".format(loci)), 'w') as clustsout:
                                LOGGER.debug("writing chunk - seqsize {} loci {} {}".format(seqsize, loci, clustsout.name))
                                clustsout.write("\n//\n//\n".join(seqlist)+"\n//\n//\n")
                            ## reset list and counter
                            seqlist = []
                            seqsize = 0

                    ## store the new seed on top of fseq
                    fseqs.append(">{}\n{}".format(seed, allcons[seed]))
                    lastseed = seed

                ## add match to the seed
                seq = allcons[hit]
                ## revcomp if orientation is reversed
                if ori == "-":
                    seq = fullcomp(seq)[::-1]
                fseqs.append(">{}\n{}".format(hit, seq))
            except KeyError as inst:
                ## Caught bad seed or hit? Log and continue.
                LOGGER.error("Bad Seed/Hit: seqsize {}\tloci {}\tseed {}\thit {}".format(seqsize, loci, seed, hit))

    ## write whatever is left over to the clusts file
    if fseqs:
        seqlist.append("\n".join(fseqs))
        seqsize += 1
        loci += seqsize
    if seqlist:
        with open(os.path.join(data.tmpdir,
            data.name+".chunk_{}".format(loci)), 'w') as clustsout:
            clustsout.write("\n//\n//\n".join(seqlist)+"\n//\n//\n")

    ## final progress and cleanup
    del allcons
    clustbits = glob.glob(os.path.join(data.tmpdir, data.name+".chunk_*"))

    ## return stuff
    return clustbits, loci

Response: This function is a subfunction of build_clustbits that creates a bit - list of all the unaligned clusters. It will also build the bit - list of all the unaligned clusters.


Instruction:
Make a summary of the following Python 3 code
def build_input_file(data, samples, randomseed):

    ## get all of the consens handles for samples that have consens reads
    ## this is better than using sample.files.consens for selecting files
    ## b/c if they were moved we only have to edit data.dirs.consens

    ## scratch the statement above, people shouldn't be moving files,
    ## they should be using merge/branch, and so sample.files.consens
    ## is needed to keep track of samples from different dirs if they
    ## are later merged into the same assembly.
    #conshandles = [os.path.join(data.dirs.consens, sample.name+".consens.gz") \
    #              for sample in samples if \
    #              sample.stats.reads_consens]
    conshandles = [sample.files.consens[0] \
                  for sample in samples if \
                  sample.stats.reads_consens]
    conshandles.sort()
    assert conshandles, "no consensus files found"

    ## concatenate all of the gzipped consens files
    cmd = ['cat'] + conshandles
    #allcons = os.path.join(data.dirs.consens, data.name+"_catcons.tmp")
    allcons = os.path.join(data.dirs.across, data.name+"_catcons.tmp")
    LOGGER.debug(" ".join(cmd))
    with open(allcons, 'w') as output:
        call = sps.Popen(cmd, stdout=output, close_fds=True)
        call.communicate()

    ## a string of sed substitutions for temporarily replacing hetero sites
    ## skips lines with '>', so it doesn't affect taxon names
    subs = ["/>/!s/W/A/g", "/>/!s/w/A/g", "/>/!s/R/A/g", "/>/!s/r/A/g",
            "/>/!s/M/A/g", "/>/!s/m/A/g", "/>/!s/K/T/g", "/>/!s/k/T/g",
            "/>/!s/S/C/g", "/>/!s/s/C/g", "/>/!s/Y/C/g", "/>/!s/y/C/g"]
    subs = ";".join(subs)

    ## impute pseudo-haplo information to avoid mismatch at hetero sites
    ## the read data with hetero sites is put back into clustered data later.
    ## pipe passed data from gunzip to sed.
    cmd1 = ["gunzip", "-c", allcons]
    cmd2 = ["sed", subs]
    LOGGER.debug(" ".join(cmd1))
    proc1 = sps.Popen(cmd1, stdout=sps.PIPE, close_fds=True)
    allhaps = allcons.replace("_catcons.tmp", "_cathaps.tmp")
    with open(allhaps, 'w') as output:
        LOGGER.debug(" ".join(cmd2))
        proc2 = sps.Popen(cmd2, stdin=proc1.stdout, stdout=output, close_fds=True)
        proc2.communicate()
    proc1.stdout.close()

    ## now sort the file using vsearch
    allsort = allcons.replace("_catcons.tmp", "_catsort.tmp")
    cmd1 = [ipyrad.bins.vsearch,
            "--sortbylength", allhaps,
            "--fasta_width", "0",
            "--output", allsort]
    LOGGER.debug(" ".join(cmd1))
    proc1 = sps.Popen(cmd1, close_fds=True)
    proc1.communicate()

    ## shuffle sequences within size classes. Tested seed (8/31/2016)
    ## shuffling works repeatably with seed.
    random.seed(randomseed)

    ## open an iterator to lengthsorted file and grab two lines at at time
    allshuf = allcons.replace("_catcons.tmp", "_catshuf.tmp")
    outdat = open(allshuf, 'w')
    indat = open(allsort, 'r')
    idat = itertools.izip(iter(indat), iter(indat))
    done = 0

    chunk = [idat.next()]
    while not done:
        ## grab 2-lines until they become shorter (unless there's only one)
        oldlen = len(chunk[-1][-1])
        while 1:
            try:
                dat = idat.next()
            except StopIteration:
                done = 1
                break
            if len(dat[-1]) == oldlen:
                chunk.append(dat)
            else:
                ## send the last chunk off to be processed
                random.shuffle(chunk)
                outdat.write("".join(itertools.chain(*chunk)))
                ## start new chunk
                chunk = [dat]
                break

    ## do the last chunk
    random.shuffle(chunk)
    outdat.write("".join(itertools.chain(*chunk)))

    indat.close()
    outdat.close()

Response: Builds the input file for the consensus files.


Instruction:
Write a Python 3 script to
step 6-1: Clears dirs and databases and calls 'build_input_file()'

Response:
def clean_and_build_concat(data, samples, randomseed, ipyclient):
    """
    STEP 6-1:
    Clears dirs and databases and calls 'build_input_file()'
    """
    ## but check for new clust database name if this is a new branch
    cleanup_tempfiles(data)
    catclust = os.path.join(data.dirs.across, data.name+"_catclust.gz")
    if os.path.exists(catclust):
        os.remove(catclust)
    if os.path.exists(data.clust_database):
        os.remove(data.clust_database)

    ## get parallel view
    start = time.time()
    printstr = " concat/shuffle input  | {} | s6 |"

    ## make a vsearch input fasta file with all samples reads concat
    async = ipyclient[0].apply(build_input_file, *[data, samples, randomseed])
    while 1:
        ready = int(async.ready())
        elapsed = datetime.timedelta(seconds=int(time.time()-start))
        progressbar(1, ready, printstr.format(elapsed), spacer=data._spacer)
        if ready:
            break
        else:
            time.sleep(0.1)
    print("")

    ## store that this step was successful
    if not async.successful():
        raise IPyradWarningExit(async.result())


Instruction:
Here you have a function in Python 3, explain what it does
def run(data, samples, noreverse, force, randomseed, ipyclient, **kwargs):

    ## if force then set checkpoint to zero and run all substeps for just
    ## the user specified steps.
    if force:
        data._checkpoint = 0
        if kwargs.get('substeps'):
            substeps = kwargs.get('substeps')
        else:
            substeps = range(1, 8)

    ## if {data}._checkpoint attribute exists then find the checkpoint where
    ## this assembly left off (unless force) and build step list from there.
    else:
        if kwargs.get('substeps'):
            substeps = kwargs.get('substeps')
        else:
            if hasattr(data, '_checkpoint'):
                substeps = range(max(1, data._checkpoint), 8)
            else:
                data._checkpoint = 0
                substeps = range(1, 8)

    ## build substeps list to subset which funtions need to be run
    if isinstance(substeps, (int, float, str)):
        substeps = [substeps]
        substeps = [int(i) for i in substeps]

    ## print continuation message
    if substeps[0] != 1:
        print("{}Continuing from checkpoint 6.{}"\
              .format(data._spacer, substeps[0]))
    LOGGER.info("checkpoint = %s", data._checkpoint)
    LOGGER.info("substeps = %s", substeps)

    ## Set variables on data that are needed for all steps;
    data.dirs.across = os.path.realpath(
        os.path.join(data.paramsdict["project_dir"], data.name+"_across"))
    data.tmpdir = os.path.join(data.dirs.across, data.name+"-tmpalign")
    data.clust_database = os.path.join(data.dirs.across, data.name+".clust.hdf5")
    if not os.path.exists(data.dirs.across):
        os.mkdir(data.dirs.across)
    if not os.path.exists(data.tmpdir):
        os.mkdir(data.tmpdir)
    data.cpus = data._ipcluster["cores"]
    if not data.cpus:
        data.cpus = len(ipyclient)

    ## STEP 6-1: Clean database and build input concat file for clustering
    if 1 in substeps:
        clean_and_build_concat(data, samples, randomseed, ipyclient)
        data._checkpoint = 1

    ## STEP 6-2: Cluster across w/ vsearch; uses all threads on largest host
    if 2 in substeps:
        call_cluster(data, noreverse, ipyclient)
        data._checkpoint = 2

    ## builds consens cluster bits and writes them to the tmp directory. These
    ## will not be deleted until either step 6-6 is complete, or the force flag
    ## is used. This will clear the tmpdir if it is run.
    if 3 in substeps:
        build_clustbits(data, ipyclient, force)
        data._checkpoint = 3

    ## muscle align the cluster bits and create tmp hdf5 indel arrays for the
    ## next step. These will not be deleted until...
    if 4 in substeps:
        multi_muscle_align(data, samples, ipyclient)
        data._checkpoint = 4

    ## fill the indel array with the indel tmp arrays from aligning step.
    if 5 in substeps:
        build_indels(data, samples, ipyclient)
        data._checkpoint = 5

    if 6 in substeps:
        ## builds the final HDF5 array which includes three main keys
        ## /catg -- contains all indiv catgs and has indels inserted
        ##   .attr['samples'] = [samples]
        ## /filters -- filled for dups, left empty for others until step 7.
        ##   .attr['filters'] = [f1, f2, f3, f4, f5]
        ## /seqs -- contains the clustered sequence data as string arrays
        ##   .attr['samples'] = [samples]
        ## /edges -- gets the paired split locations for now.
        ## /snps  -- left empty for now

        ## FILL SUPERCATG and fills dupfilter, indfilter, and nalleles
        ## this function calls singlecat() on each sample and enters their
        ## resulting arrays into the superarray. If all singlecats are built
        ## then it will continue to enter them into the database.
        LOGGER.info("multicat -- building full database")
        new_multicat(data, samples, ipyclient)
        data._checkpoint = 6

    if 7 in substeps:
        ## FILL SUPERSEQS and fills edges(splits) for paired-end data
        fill_superseqs(data, samples)
        data._checkpoint = 7

        ## remove files but not dir (used in step 1 too)
        cleanup_tempfiles(data)
        ## remove the tmpdir
        if os.path.exists(data.tmpdir):
            shutil.rmtree(data.tmpdir)

        ## set sample states
        for sample in samples:
            sample.stats.state = 6
        print("")

Response: Run the sequence of functions in a single step.