Instruction:
Explain what the following Python 3 code does
def Copy(self, name=None):
        new = copy.copy(self)
        new.d = copy.copy(self.d)
        new.name = name if name is not None else self.name
        return new

Response: Returns a shallow copy of the object.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def Scale(self, factor):
        new = self.Copy()
        new.d.clear()

        for val, prob in self.Items():
            new.Set(val * factor, prob)
        return new

Response: Multiplies the values by a factor.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def Log(self, m=None):
        if self.log:
            raise ValueError("Pmf/Hist already under a log transform")
        self.log = True

        if m is None:
            m = self.MaxLike()

        for x, p in self.d.iteritems():
            if p:
                self.Set(x, math.log(p / m))
            else:
                self.Remove(x)

Response: Log transforms the probabilities.


Instruction:
Implement a Python 3 function for
printing the values and freqs and probabilities in ascending order.

Response:
def Print(self):
        """Prints the values and freqs/probs in ascending order."""
        for val, prob in sorted(self.d.iteritems()):
            print(val, prob)


Instruction:
Implement a Python 3 function for
incrementing the freq / prob associated with the value x.

Response:
def Incr(self, x, term=1):
        """Increments the freq/prob associated with the value x.

        Args:
            x: number value
            term: how much to increment by
        """
        self.d[x] = self.d.get(x, 0) + term


Instruction:
Write a Python 3 function that can
multiply the freq / prob associated with the value x.

Response:
def Mult(self, x, factor):
        """Scales the freq/prob associated with the value x.

        Args:
            x: number value
            factor: how much to multiply by
        """
        self.d[x] = self.d.get(x, 0) * factor


Instruction:
Given the following Python 3 function, write the documentation
def IsSubset(self, other):
        for val, freq in self.Items():
            if freq > other.Freq(val):
                return False
        return True

Response: Checks whether the values in this histogram are a subset of the values in the given histogram.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def Subtract(self, other):
        for val, freq in other.Items():
            self.Incr(val, -freq)

Response: Subtracts the values in the given histogram from this histogram.


Instruction:
Can you create a Python 3 function that
returns the probability that a sample from this Pmf exceeds x.

Response:
def ProbGreater(self, x):
        """Probability that a sample from this Pmf exceeds x.

        x: number

        returns: float probability
        """
        t = [prob for (val, prob) in self.d.iteritems() if val > x]
        return sum(t)


Instruction:
How would you implement a function in Python 3 that
normalizes this PMF so the sum of all probs is fraction.

Response:
def Normalize(self, fraction=1.0):
        """Normalizes this PMF so the sum of all probs is fraction.

        Args:
            fraction: what the total should be after normalization

        Returns: the total probability before normalizing
        """
        if self.log:
            raise ValueError("Pmf is under a log transform")

        total = self.Total()
        if total == 0.0:
            raise ValueError('total probability is zero.')
            logging.warning('Normalize: total probability is zero.')
            return total

        factor = float(fraction) / total
        for x in self.d:
            self.d[x] *= factor

        return total


Instruction:
How would you explain what the following Python 3 function does
def Random(self):
        if len(self.d) == 0:
            raise ValueError('Pmf contains no values.')

        target = random.random()
        total = 0.0
        for x, p in self.d.iteritems():
            total += p
            if total >= target:
                return x

        # we shouldn't get here
        assert False

Response: Chooses a random element from this PMF.


Instruction:
Explain what the following Python 3 code does
def Mean(self):
        mu = 0.0
        for x, p in self.d.iteritems():
            mu += p * x
        return mu

Response: Computes the mean of a PMF.


Instruction:
Write a Python 3 function for
computing the variance of a PMF.

Response:
def Var(self, mu=None):
        """Computes the variance of a PMF.

        Args:
            mu: the point around which the variance is computed;
                if omitted, computes the mean

        Returns:
            float variance
        """
        if mu is None:
            mu = self.Mean()

        var = 0.0
        for x, p in self.d.iteritems():
            var += p * (x - mu) ** 2
        return var


Instruction:
Can you implement a function in Python 3 that
returns the value with the highest probability.

Response:
def MaximumLikelihood(self):
        """Returns the value with the highest probability.

        Returns: float probability
        """
        prob, val = max((prob, val) for val, prob in self.Items())
        return val


Instruction:
Write a Python 3 function that can
compute the Pmf of the sum of values drawn from self and other.

Response:
def AddPmf(self, other):
        """Computes the Pmf of the sum of values drawn from self and other.

        other: another Pmf

        returns: new Pmf
        """
        pmf = Pmf()
        for v1, p1 in self.Items():
            for v2, p2 in other.Items():
                pmf.Incr(v1 + v2, p1 * p2)
        return pmf


Instruction:
Create a Python 3 function to
compute the Pmf of the sum a constant and values from self and other.

Response:
def AddConstant(self, other):
        """Computes the Pmf of the sum a constant and  values from self.

        other: a number

        returns: new Pmf
        """
        pmf = Pmf()
        for v1, p1 in self.Items():
            pmf.Set(v1 + other, p1)
        return pmf


Instruction:
How would you code a function in Python 3 to
compute the CDF of the maximum of k selections from this distribution.

Response:
def Max(self, k):
        """Computes the CDF of the maximum of k selections from this dist.

        k: int

        returns: new Cdf
        """
        cdf = self.MakeCdf()
        cdf.ps = [p ** k for p in cdf.ps]
        return cdf


Instruction:
Write a Python 3 function for
getting the marginal distribution of the indicated variable.

Response:
def Marginal(self, i, name=''):
        """Gets the marginal distribution of the indicated variable.

        i: index of the variable we want

        Returns: Pmf
        """
        pmf = Pmf(name=name)
        for vs, prob in self.Items():
            pmf.Incr(vs[i], prob)
        return pmf


Instruction:
Can you generate the documentation for the following Python 3 function
def Conditional(self, i, j, val, name=''):
        pmf = Pmf(name=name)
        for vs, prob in self.Items():
            if vs[j] != val: continue
            pmf.Incr(vs[i], prob)

        pmf.Normalize()
        return pmf

Response: Gets the conditional distribution of the indicated variable.


Instruction:
Given the following Python 3 function, write the documentation
def MaxLikeInterval(self, percentage=90):
        interval = []
        total = 0

        t = [(prob, val) for val, prob in self.Items()]
        t.sort(reverse=True)

        for prob, val in t:
            interval.append(val)
            total += prob
            if total >= percentage / 100.0:
                break

        return interval

Response: Returns the maximum - likelihood credible interval.


Instruction:
Can you write a function in Python 3 where it
returns a copy of this Cdf.

Response:
def Copy(self, name=None):
        """Returns a copy of this Cdf.

        Args:
            name: string name for the new Cdf
        """
        if name is None:
            name = self.name
        return Cdf(list(self.xs), list(self.ps), name)


Instruction:
Here you have a function in Python 3, explain what it does
def Append(self, x, p):
        self.xs.append(x)
        self.ps.append(p)

Response: Add an x and p pair to the end of this CDF.


Instruction:
Here you have a function in Python 3, explain what it does
def Shift(self, term):
        new = self.Copy()
        new.xs = [x + term for x in self.xs]
        return new

Response: Adds a term to the xs.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def Scale(self, factor):
        new = self.Copy()
        new.xs = [x * factor for x in self.xs]
        return new

Response: Multiplies the xs by a factor.


Instruction:
Write a Python 3 function that can
return CDF ( x ) the probability that corresponds to value x.

Response:
def Prob(self, x):
        """Returns CDF(x), the probability that corresponds to value x.

        Args:
            x: number

        Returns:
            float probability
        """
        if x < self.xs[0]: return 0.0
        index = bisect.bisect(self.xs, x)
        p = self.ps[index - 1]
        return p


Instruction:
Given the following Python 3 function, write the documentation
def Value(self, p):
        if p < 0 or p > 1:
            raise ValueError('Probability p must be in range [0, 1]')

        if p == 0: return self.xs[0]
        if p == 1: return self.xs[-1]
        index = bisect.bisect(self.ps, p)
        if p == self.ps[index - 1]:
            return self.xs[index - 1]
        else:
            return self.xs[index]

Response: Returns InverseCDF ( p ) the value that corresponds to probability p.


Instruction:
Here you have a function in Python 3, explain what it does
def Mean(self):
        old_p = 0
        total = 0.0
        for x, new_p in zip(self.xs, self.ps):
            p = new_p - old_p
            total += p * x
            old_p = new_p
        return total

Response: Computes the mean of a CDF.


Instruction:
Write a Python 3 function for
computing the central credible interval.

Response:
def CredibleInterval(self, percentage=90):
        """Computes the central credible interval.

        If percentage=90, computes the 90% CI.

        Args:
            percentage: float between 0 and 100

        Returns:
            sequence of two floats, low and high
        """
        prob = (1 - percentage / 100.0) / 2
        interval = self.Value(prob), self.Value(1 - prob)
        return interval


Instruction:
Can you generate the documentation for the following Python 3 function
def Render(self):
        xs = [self.xs[0]]
        ps = [0.0]
        for i, p in enumerate(self.ps):
            xs.append(self.xs[i])
            ps.append(p)

            try:
                xs.append(self.xs[i + 1])
                ps.append(p)
            except IndexError:
                pass
        return xs, ps

Response: Generates a sequence of points suitable for plotting.


Instruction:
Create a Python 3 function for
computing the CDF of the maximum of k selections from this distribution.

Response:
def Max(self, k):
        """Computes the CDF of the maximum of k selections from this dist.

        k: int

        returns: new Cdf
        """
        cdf = self.Copy()
        cdf.ps = [p ** k for p in cdf.ps]
        return cdf


Instruction:
Make a summary of the following Python 3 code
def LogUpdate(self, data):
        for hypo in self.Values():
            like = self.LogLikelihood(data, hypo)
            self.Incr(hypo, like)

Response: Updates a suite of hypotheses based on new data.


Instruction:
Implement a Python 3 function for
updating each hypothesis in the set based on the dataset.

Response:
def UpdateSet(self, dataset):
        """Updates each hypothesis based on the dataset.

        This is more efficient than calling Update repeatedly because
        it waits until the end to Normalize.

        Modifies the suite directly; if you want to keep the original, make
        a copy.

        dataset: a sequence of data

        returns: the normalizing constant
        """
        for data in dataset:
            for hypo in self.Values():
                like = self.Likelihood(data, hypo)
                self.Mult(hypo, like)
        return self.Normalize()


Instruction:
How would you code a function in Python 3 to
print the hypotheses and their probabilities.

Response:
def Print(self):
        """Prints the hypotheses and their probabilities."""
        for hypo, prob in sorted(self.Items()):
            print(hypo, prob)


Instruction:
Given the following Python 3 function, write the documentation
def MakeOdds(self):
        for hypo, prob in self.Items():
            if prob:
                self.Set(hypo, Odds(prob))
            else:
                self.Remove(hypo)

Response: Transforms from probabilities to odds.


Instruction:
Write a Python 3 function for
transforming from odds to probabilities.

Response:
def MakeProbs(self):
        """Transforms from odds to probabilities."""
        for hypo, odds in self.Items():
            self.Set(hypo, Probability(odds))


Instruction:
Here you have a function in Python 3, explain what it does
def MakePmf(self, xs, name=''):
        pmf = Pmf(name=name)
        for x in xs:
            pmf.Set(x, self.Density(x))
        pmf.Normalize()
        return pmf

Response: Makes a discrete version of this Pdf evaluated at xs.


Instruction:
Explain what the following Python 3 code does
def Update(self, data):
        heads, tails = data
        self.alpha += heads
        self.beta += tails

Response: Updates a Beta distribution.


Instruction:
Write a Python 3 function that can
generate a random sample from this distribution.

Response:
def Sample(self, n):
        """Generates a random sample from this distribution.

        n: int sample size
        """
        size = n,
        return numpy.random.beta(self.alpha, self.beta, size)


Instruction:
Here you have a function in Python 3, explain what it does
def MakePmf(self, steps=101, name=''):
        if self.alpha < 1 or self.beta < 1:
            cdf = self.MakeCdf()
            pmf = cdf.MakePmf()
            return pmf

        xs = [i / (steps - 1.0) for i in xrange(steps)]
        probs = [self.EvalPdf(x) for x in xs]
        pmf = MakePmfFromDict(dict(zip(xs, probs)), name)
        return pmf

Response: Returns a Pmf of this distribution.


Instruction:
Write a Python 3 function that can
return the CDF of this distribution.

Response:
def MakeCdf(self, steps=101):
        """Returns the CDF of this distribution."""
        xs = [i / (steps - 1.0) for i in xrange(steps)]
        ps = [scipy.special.betainc(self.alpha, self.beta, x) for x in xs]
        cdf = Cdf(xs, ps)
        return cdf


Instruction:
Given the following Python 3 function, write the documentation
def Update(self, data):
        m = len(data)
        self.params[:m] += data

Response:
Updates a Dirichlet distribution.
        data : sequence of observations in order corresponding to params


Instruction:
Create a Python 3 function for
generating a random variate from this distribution.

Response:
def Random(self):
        """Generates a random variate from this distribution.

        Returns: normalized vector of fractions
        """
        p = numpy.random.gamma(self.params)
        return p / p.sum()


Instruction:
Can you create a Python 3 function that
computes the likelihood of the data.

Response:
def Likelihood(self, data):
        """Computes the likelihood of the data.

        Selects a random vector of probabilities from this distribution.

        Returns: float probability
        """
        m = len(data)
        if self.n < m:
            return 0

        x = data
        p = self.Random()
        q = p[:m] ** x
        return q.prod()


Instruction:
Can you implement a function in Python 3 that
computes the log likelihood of the data.

Response:
def LogLikelihood(self, data):
        """Computes the log likelihood of the data.

        Selects a random vector of probabilities from this distribution.

        Returns: float log probability
        """
        m = len(data)
        if self.n < m:
            return float('-inf')

        x = self.Random()
        y = numpy.log(x[:m]) * data
        return y.sum()


Instruction:
Can you generate a brief explanation for the following Python 3 code
def MarginalBeta(self, i):
        alpha0 = self.params.sum()
        alpha = self.params[i]
        return Beta(alpha, alpha0 - alpha)

Response: Computes the marginal distribution of the ith element.


Instruction:
Can you generate the documentation for the following Python 3 function
def PredictivePmf(self, xs, name=''):
        alpha0 = self.params.sum()
        ps = self.params / alpha0
        return MakePmfFromItems(zip(xs, ps), name=name)

Response: Makes a predictive distribution.


Instruction:
Write a Python 3 function that can
return the annotation for the given databases and features.

Response:
def _get_ann(dbs, features):
    """
    Gives format to annotation for html table output
    """
    value = ""
    for db, feature in zip(dbs, features):
        value += db + ":" + feature
    return value


Instruction:
Explain what the following Python 3 code does
def make_profile(data, out_dir, args):
    safe_dirs(out_dir)
    main_table = []
    header = ['id', 'ann']
    n = len(data[0])
    bar = ProgressBar(maxval=n)
    bar.start()
    bar.update(0)
    for itern, c in enumerate(data[0]):
        bar.update(itern)
        logger.debug("creating cluser: {}".format(c))
        safe_dirs(os.path.join(out_dir, c))
        valid, ann, pos_structure = _single_cluster(c, data, os.path.join(out_dir, c, "maps.tsv"), args)
        data[0][c].update({'profile': pos_structure})
        loci = data[0][c]['loci']
        data[0][c]['precursor'] = {"seq": precursor_sequence(loci[0][0:5], args.ref)}
        data[0][c]['precursor']["colors"] = _parse(data[0][c]['profile'], data[0][c]['precursor']["seq"])
        data[0][c]['precursor'].update(run_rnafold(data[0][c]['precursor']['seq']))

    return data

Response: Make data report for each cluster


Instruction:
Make a summary of the following Python 3 code
def _expand(dat, counts, start, end):
    for pos in range(start, end):
        for s in counts:
            dat[s][pos] += counts[s]
    return dat

Response: expand the same counts from start to end


Instruction:
Write a Python 3 function for
converting data frame into pandas DataFrame

Response:
def _convert_to_df(in_file, freq, raw_file):
    """
    convert data frame into table with pandas
    """
    dat = defaultdict(Counter)
    if isinstance(in_file, (str, unicode)):
        with open(in_file) as in_handle:
            for line in in_handle:
                cols = line.strip().split("\t")
                counts = freq[cols[3]]
                dat = _expand(dat, counts, int(cols[1]), int(cols[2]))
    else:
        if raw_file:
            out_handle = open(raw_file, "w")
        for name in in_file:
            counts = freq[name]
            if raw_file:
                print("%s\t%s\t%s\t%s\t%s\t%s" % ("chr", in_file[name][0], in_file[name][1], name, sum(counts.values()), "+"), file=out_handle, end="")
            dat = _expand(dat, counts, in_file[name][0], in_file[name][1])

    for s in dat:
        for p in dat[s]:
            dat[s][p] = mlog2(dat[s][p] + 1)
    return dat


Instruction:
Can you tell what is the following Python 3 function doing
def _make(c):
    ann = defaultdict(list)

    for pos in c['ann']:
        for db in pos:
            ann[db] += list(pos[db])
    logger.debug(ann)

    valid = [l for l in c['valid']]
    ann_list = [", ".join(list(set(ann[feature]))) for feature in ann if feature in valid]

    return valid, ann_list

Response: create html from template adding figure and sequence counts counts


Instruction:
Can you generate the documentation for the following Python 3 function
def _single_cluster(c, data, out_file, args):
    valid, ann = 0, 0
    raw_file = None
    freq = defaultdict()
    [freq.update({s.keys()[0]: s.values()[0]}) for s in data[0][c]['freq']]
    names = [s.keys()[0] for s in data[0][c]['seqs']]
    seqs = [s.values()[0] for s in data[0][c]['seqs']]
    loci = data[0][c]['loci']

    if loci[0][3] - loci[0][2] > 500:
        logger.info("locus bigger > 500 nt, skipping: %s" % loci)
        return valid, ann, {}
    if not file_exists(out_file):
        if args.razer:
            logger.debug("map with razer all sequences to all loci %s " % loci)
            map_to_precursors(seqs, names, {loci[0][0]: [loci[0][0:5]]}, out_file, args)
        else:
            logger.debug("map with biopython fn all sequences to all loci %s " % loci)
            if args.debug:
                raw_file = out_file
            out_file = map_to_precursor_biopython(seqs, names, loci[0][0:5], args)

    logger.debug("plot sequences on loci")
    df = _convert_to_df(out_file, freq, raw_file)
    if df:
        valid, ann = _make(data[0][c])

    return valid, ann, df

Response: Map sequences on precursors and create


Instruction:
Can you create a Python 3 function that
reads json cluster and populate as cluster class

Response:
def read_cluster(data, id=1):
    """Read json cluster and populate as cluster class"""
    cl = cluster(1)

    # seqs = [s.values()[0] for s in data['seqs']]
    names = [s.keys()[0] for s in data['seqs']]
    cl.add_id_member(names, 1)
    freq = defaultdict()
    [freq.update({s.keys()[0]: s.values()[0]}) for s in data['freq']]


Instruction:
Can you write a function in Python 3 where it
writes json file from seqcluster cluster

Response:
def write_data(data, out_file):
    """write json file from seqcluster cluster"""
    with open(out_file, 'w') as handle_out:
        handle_out.write(json.dumps([data], skipkeys=True, indent=2))


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_sequences_from_cluster(c1, c2, data):
    seqs1 = data[c1]['seqs']
    seqs2 = data[c2]['seqs']
    seqs = list(set(seqs1 + seqs2))
    names = []
    for s in seqs:
        if s in seqs1 and s in seqs2:
            names.append("both")
        elif s in seqs1:
            names.append(c1)
        else:
            names.append(c2)
    return seqs, names

Response: get all sequences from on cluster


Instruction:
Can you write a function in Python 3 where it
maps sequences to precursors with razers3

Response:
def map_to_precursors(seqs, names, loci, out_file, args):
    """map sequences to precursors with razers3"""
    with make_temp_directory() as temp:
        pre_fasta = os.path.join(temp, "pre.fa")
        seqs_fasta = os.path.join(temp, "seqs.fa")
        out_sam = os.path.join(temp, "out.sam")
        pre_fasta = get_loci_fasta(loci, pre_fasta, args.ref)
        out_precursor_file = out_file.replace("tsv", "fa")
        seqs_fasta = get_seqs_fasta(seqs, names, seqs_fasta)

        # print(open(pre_fasta).read().split("\n")[1])
        if find_cmd("razers3"):
            cmd = "razers3 -dr 2 -i 80 -rr 90 -f -o {out_sam} {temp}/pre.fa  {seqs_fasta}"
            run(cmd.format(**locals()))
            out_file = read_alignment(out_sam, loci, seqs, out_file)
            shutil.copy(pre_fasta, out_precursor_file)
    return out_file


Instruction:
How would you code a function in Python 3 to
get sequence from genome

Response:
def precursor_sequence(loci, reference):
    """Get sequence from genome"""
    region = "%s\t%s\t%s\t.\t.\t%s" % (loci[1], loci[2], loci[3], loci[4])
    precursor = pybedtools.BedTool(str(region), from_string=True).sequence(fi=reference, s=True)
    return open(precursor.seqfn).read().split("\n")[1]


Instruction:
How would you code a function in Python 3 to
map sequences to precursors with franpr algorithm to avoid writting on disk

Response:
def map_to_precursors_on_fly(seqs, names, loci, args):
    """map sequences to precursors with franpr algorithm to avoid writting on disk"""
    precursor = precursor_sequence(loci, args.ref).upper()
    dat = dict()
    for s, n in itertools.izip(seqs, names):
        res = pyMatch.Match(precursor, str(s), 1, 3)
        if res > -1:
            dat[n] = [res, res + len(s)]
    logger.debug("mapped in %s: %s out of %s" % (loci, len(dat), len(seqs)))
    return dat


Instruction:
Given the following Python 3 function, write the documentation
def _align(x, y, local = False):
    if local:
        aligned_x = pairwise2.align.localxx(x, y)
    else:
        aligned_x =  pairwise2.align.globalms(x, y, 1, -1, -1, -0.5)

    if aligned_x:
        sorted_alignments = sorted(aligned_x, key=operator.itemgetter(2))
        e = enumerate(sorted_alignments[0][0])
        nts = [i for i,c in e if c != "-"]
        return [min(nts), max(nts)]

Response: aligns x y with local or global


Instruction:
Explain what the following Python 3 code does
def map_to_precursor_biopython(seqs, names, loci, args):
    precursor = precursor_sequence(loci, args.ref).upper()
    dat = dict()
    for s, n in itertools.izip(seqs, names):
        res = _align(str(s), precursor)
        if res:
            dat[n] = res
    logger.debug("mapped in %s: %s out of %s" % (loci, len(dat), len(seqs)))
    return dat

Response: map the sequences to the precursor sequence


Instruction:
How would you explain what the following Python 3 function does
def get_seqs_fasta(seqs, names, out_fa):
    with open(out_fa, 'w') as fa_handle:
        for s, n in itertools.izip(seqs, names):
            print(">cx{1}-{0}\n{0}".format(s, n), file=fa_handle)
    return out_fa

Response: get fasta from sequences


Instruction:
Here you have a function in Python 3, explain what it does
def get_loci_fasta(loci, out_fa, ref):
    if not find_cmd("bedtools"):
        raise ValueError("Not bedtools installed")
    with make_temp_directory() as temp:
        bed_file = os.path.join(temp, "file.bed")
        for nc, loci in loci.iteritems():
            for l in loci:
                with open(bed_file, 'w') as bed_handle:
                    logger.debug("get_fasta: loci %s" % l)
                    nc, c, s, e, st = l
                    print("{0}\t{1}\t{2}\t{3}\t{3}\t{4}".format(c, s, e, nc, st), file=bed_handle)
                get_fasta(bed_file, ref, out_fa)
    return out_fa

Response: get fasta from precursor


Instruction:
Given the following Python 3 function, write the documentation
def read_alignment(out_sam, loci, seqs, out_file):
    hits = defaultdict(list)
    with open(out_file, "w") as out_handle:
        samfile = pysam.Samfile(out_sam, "r")
        for a in samfile.fetch():
            if not a.is_unmapped:
                nm = int([t[1] for t in a.tags if t[0] == "NM"][0])
                a = makeBED(a)
                if not a:
                    continue
                ref, locus = get_loci(samfile.getrname(int(a.chr)), loci)
                hits[a.name].append((nm, "%s %s %s %s %s %s" % (a.name, a.name.split("-")[0], locus, ref, a.start, a.end)))
        for hit in hits.values():
            nm = hit[0][0]
            for l in hit:
                if nm == l[0]:
                    print(l[1], file=out_handle)
    return out_file

Response:
read which seqs map to which loci and
    return a tab separated file


Instruction:
Given the following Python 3 function, write the documentation
def _download_mirbase(args, version="CURRENT"):
    if not args.hairpin or not args.mirna:
        logger.info("Working with version %s" % version)
        hairpin_fn = op.join(op.abspath(args.out), "hairpin.fa.gz")
        mirna_fn = op.join(op.abspath(args.out), "miRNA.str.gz")
        if not file_exists(hairpin_fn):
            cmd_h = "wget ftp://mirbase.org/pub/mirbase/%s/hairpin.fa.gz -O %s &&  gunzip -f !$" % (version, hairpin_fn)
            do.run(cmd_h, "download hairpin")
        if not file_exists(mirna_fn):
            cmd_m = "wget ftp://mirbase.org/pub/mirbase/%s/miRNA.str.gz -O %s && gunzip -f !$" % (version, mirna_fn)
            do.run(cmd_m, "download mirna")
    else:
        return args.hairpin, args.mirna

Response: Download files from mirbase if not present


Instruction:
Explain what the following Python 3 code does
def _make_unique(name, idx):
    p = re.compile(".[aA-zZ]+_x[0-9]+")
    if p.match(name):
        tags = name[1:].split("_x")
        return ">%s_%s_x%s" % (tags[0], idx, tags[1])
    return name.replace("@", ">")

Response: Make name unique in case only counts there


Instruction:
Write a Python 3 function that can
convert names of sequences to unique ids

Response:
def _filter_seqs(fn):
    """Convert names of sequences to unique ids"""
    out_file = op.splitext(fn)[0] + "_unique.fa"
    idx = 0
    if not file_exists(out_file):
        with open(out_file, 'w') as out_handle:
            with open(fn) as in_handle:
                for line in in_handle:
                    if line.startswith("@") or line.startswith(">"):
                        fixed_name = _make_unique(line.strip(), idx)
                        seq = in_handle.next().strip()
                        counts = _get_freq(fixed_name)
                        if len(seq) < 26 and (counts > 1 or counts == 0):
                            idx += 1
                            print(fixed_name, file=out_handle, end="\n")
                            print(seq, file=out_handle, end="\n")
                        if line.startswith("@"):
                            in_handle.next()
                            in_handle.next()
    return out_file


Instruction:
Implement a function in Python 3 to
read the precursor file for that species and return a dictionary of hairpin names to species.

Response:
def _read_precursor(precursor, sps):
    """
    Load precursor file for that species
    """
    hairpin = defaultdict(str)
    name = None
    with open(precursor) as in_handle:
        for line in in_handle:
            if line.startswith(">"):
                if hairpin[name]:
                    hairpin[name] = hairpin[name] + "NNNNNNNNNNNN"
                name = line.strip().replace(">", " ").split()[0]
            else:
                hairpin[name] += line.strip()
        hairpin[name] = hairpin[name] + "NNNNNNNNNNNN"
    return hairpin


Instruction:
How would you code a function in Python 3 to
read the GTF file and return a dictionary of precursor positions on the genome.

Response:
def _read_gtf(gtf):
    """
    Load GTF file with precursor positions on genome
    """
    if not gtf:
        return gtf
    db = defaultdict(list)
    with open(gtf) as in_handle:
        for line in in_handle:
            if line.startswith("#"):
                continue
            cols = line.strip().split("\t")
            name = [n.split("=")[1] for n in cols[-1].split(";") if n.startswith("Name")]
            chrom, start, end, strand = cols[0], cols[3], cols[4], cols[6]
            if cols[2] == "miRNA_primary_transcript":
                db[name[0]].append([chrom, int(start), int(end), strand])
    return db


Instruction:
Explain what the following Python 3 code does
def _coord(sequence, start, mirna, precursor, iso):
    dif = abs(mirna[0] - start)
    if start < mirna[0]:
        iso.t5 = sequence[:dif].upper()
    elif start > mirna[0]:
        iso.t5 = precursor[mirna[0] - 1:mirna[0] - 1 + dif].lower()
    elif start == mirna[0]:
        iso.t5 = "NA"
    if dif > 4:
        logger.debug("start > 3 %s %s %s %s %s" % (start, len(sequence), dif, mirna, iso.format()))
        return None

    end = start + (len(sequence) - len(iso.add)) - 1
    dif = abs(mirna[1] - end)
    if iso.add:
        sequence = sequence[:-len(iso.add)]
    # if dif > 3:
    #    return None
    if end > mirna[1]:
        iso.t3 = sequence[-dif:].upper()
    elif end < mirna[1]:
        iso.t3 = precursor[mirna[1] - dif:mirna[1]].lower()
    elif end == mirna[1]:
        iso.t3 = "NA"
    if dif > 4:
        logger.debug("end > 3 %s %s %s %s %s" % (len(sequence), end, dif, mirna, iso.format()))
        return None
    logger.debug("%s %s %s %s %s %s" % (start, len(sequence), end, dif, mirna, iso.format()))
    return True

Response: Return a new ISO record for the given mirna.


Instruction:
Write a Python 3 function that can
annotate the sequence with the MIRNA coordinates.

Response:
def _annotate(reads, mirbase_ref, precursors):
    """
    Using SAM/BAM coordinates, mismatches and realign to annotate isomiRs
    """
    for r in reads:
        for p in reads[r].precursors:
            start = reads[r].precursors[p].start + 1  # convert to 1base
            end = start + len(reads[r].sequence)
            for mature in mirbase_ref[p]:
                mi = mirbase_ref[p][mature]
                is_iso = _coord(reads[r].sequence, start, mi, precursors[p], reads[r].precursors[p])
                logger.debug(("{r} {p} {start} {is_iso} {mature} {mi} {mature_s}").format(s=reads[r].sequence, mature_s=precursors[p][mi[0]-1:mi[1]], **locals()))
                if is_iso:
                    reads[r].precursors[p].mirna = mature
                    break
    return reads


Instruction:
Implement a function in Python 3 to
remove all hits that are not in the best match.

Response:
def _clean_hits(reads):
    """
    Select only best matches
    """
    new_reads = defaultdict(realign)
    for r in reads:
        world = {}
        sc = 0
        for p in reads[r].precursors:
            world[p] = reads[r].precursors[p].get_score(len(reads[r].sequence))
            if sc < world[p]:
                sc = world[p]
        new_reads[r] = reads[r]
        for p in world:
            logger.debug("score %s %s %s" % (r, p, world[p]))
            if sc != world[p]:
                logger.debug("remove %s %s %s" % (r, p, world[p]))
                new_reads[r].remove_precursor(p)

    return new_reads


Instruction:
Write a Python 3 function that can
read bam file and perform realignment of hits

Response:
def _read_bam(bam_fn, precursors):
    """
    read bam file and perform realignment of hits
    """
    mode = "r" if bam_fn.endswith("sam") else "rb"
    handle = pysam.Samfile(bam_fn, mode)
    reads = defaultdict(realign)
    for line in handle:
        chrom = handle.getrname(line.reference_id)
        # print("%s %s %s %s" % (line.query_name, line.reference_start, line.query_sequence, chrom))
        query_name = line.query_name
        if query_name not in reads:
            reads[query_name].sequence = line.query_sequence
        iso = isomir()
        iso.align = line
        iso.start = line.reference_start
        iso.subs, iso.add = _realign(reads[query_name].sequence, precursors[chrom], line.reference_start)
        reads[query_name].set_precursor(chrom, iso)

    reads = _clean_hits(reads)
    return reads


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _collapse_fastq(in_fn):
    args = argparse.Namespace()
    args.fastq = in_fn
    args.minimum = 1
    args.out = op.dirname(in_fn)
    return collapse_fastq(args)

Response: collapse reads into unique sequences


Instruction:
Write a Python 3 script for
reading pyMatch file and perform realignment of hits

Response:
def _read_pyMatch(fn, precursors):
    """
    read pyMatch file and perform realignment of hits
    """
    with open(fn) as handle:
        reads = defaultdict(realign)
        for line in handle:
            query_name, seq, chrom, reference_start, end, mism, add = line.split()
            reference_start = int(reference_start)
            # chrom = handle.getrname(cols[1])
            # print("%s %s %s %s" % (line.query_name, line.reference_start, line.query_sequence, chrom))
            if query_name not in reads:
                reads[query_name].sequence = seq
            iso = isomir()
            iso.align = line
            iso.start = reference_start
            iso.subs, iso.add = _realign(reads[query_name].sequence, precursors[chrom], reference_start)
            logger.debug("%s %s %s %s %s" % (query_name, reference_start, chrom, iso.subs, iso.add))
            if len(iso.subs) > 1:
                continue
            reads[query_name].set_precursor(chrom, iso)

        reads = _clean_hits(reads)
    return reads


Instruction:
Here you have a function in Python 3, explain what it does
def _parse_mut(subs):
    if subs!="0":
        subs = [[subs.replace(subs[-2:], ""),subs[-2], subs[-1]]]
    return subs

Response: Parse mutation tag from miraligner output


Instruction:
Create a Python 3 function to
read ouput of miraligner and create compatible output.

Response:
def _read_miraligner(fn):
    """Read ouput of miraligner and create compatible output."""
    reads = defaultdict(realign)
    with open(fn) as in_handle:
        in_handle.next()
        for line in in_handle:
            cols = line.strip().split("\t")
            iso = isomir()
            query_name, seq = cols[1], cols[0]
            chrom, reference_start = cols[-2], cols[3]
            iso.mirna = cols[3]
            subs, add, iso.t5, iso.t3 = cols[6:10]
            if query_name not in reads:
                reads[query_name].sequence = seq
            iso.align = line
            iso.start = reference_start
            iso.subs, iso.add = _parse_mut(subs), add
            logger.debug("%s %s %s %s %s" % (query_name, reference_start, chrom, iso.subs, iso.add))
            reads[query_name].set_precursor(chrom, iso)
    return reads


Instruction:
Given the following Python 3 function, write the documentation
def _cmd_miraligner(fn, out_file, species, hairpin, out):
    tool = _get_miraligner()
    path_db = op.dirname(op.abspath(hairpin))
    cmd = "{tool} -freq -i {fn} -o {out_file} -s {species} -db {path_db} -sub 1 -trim 3 -add 3"
    if not file_exists(out_file):
        logger.info("Running miraligner with %s" % fn)
        do.run(cmd.format(**locals()), "miraligner with %s" % fn)
        shutil.move(out_file + ".mirna", out_file)
    return out_file

Response: Run miraligner for miRNA annotation


Instruction:
Can you generate the documentation for the following Python 3 function
def _mirtop(out_files, hairpin, gff3, species, out):
    args = argparse.Namespace()
    args.hairpin = hairpin
    args.sps = species
    args.gtf = gff3
    args.add_extra = True
    args.files = out_files
    args.format = "seqbuster"
    args.out_format = "gff"
    args.out = out
    reader(args)

Response: Convert miraligner to mirtop format


Instruction:
Here you have a function in Python 3, explain what it does
def _merge(dts):
    df = pd.concat(dts)

    ma = df.pivot(index='isomir', columns='sample', values='counts')
    ma_mirna = ma
    ma = ma.fillna(0)
    ma_mirna['mirna'] = [m.split(":")[0] for m in ma.index.values]
    ma_mirna = ma_mirna.groupby(['mirna']).sum()
    ma_mirna = ma_mirna.fillna(0)
    return ma, ma_mirna

Response: merge multiple samples in one matrix


Instruction:
Write a Python 3 function for
summarizing results into single files.

Response:
def _create_counts(out_dts, out_dir):
    """Summarize results into single files."""
    ma, ma_mirna = _merge(out_dts)
    out_ma = op.join(out_dir, "counts.tsv")
    out_ma_mirna = op.join(out_dir, "counts_mirna.tsv")
    ma.to_csv(out_ma, sep="\t")
    ma_mirna.to_csv(out_ma_mirna, sep="\t")
    return out_ma_mirna, out_ma


Instruction:
Write a Python 3 script for
reading the list of files and return a list of dicts.

Response:
def miraligner(args):
    """
    Realign BAM hits to miRBAse to get better accuracy and annotation
    """
    hairpin, mirna = _download_mirbase(args)
    precursors = _read_precursor(args.hairpin, args.sps)
    matures = _read_mature(args.mirna, args.sps)
    gtf = _read_gtf(args.gtf)
    out_dts = []
    out_files = []
    for bam_fn in args.files:
        sample = op.splitext(op.basename(bam_fn))[0]
        logger.info("Reading %s" % bam_fn)
        if bam_fn.endswith("bam") or bam_fn.endswith("sam"):
            bam_fn = _sam_to_bam(bam_fn)
            bam_sort_by_n = op.splitext(bam_fn)[0] + "_sort"
            pysam.sort("-n", bam_fn, bam_sort_by_n)
            reads = _read_bam(bam_sort_by_n + ".bam", precursors)
        elif bam_fn.endswith("fasta") or bam_fn.endswith("fa") or \
                bam_fn.endswith("fastq"):
            if args.collapse:
                bam_fn = _collapse_fastq(bam_fn)
            out_file = op.join(args.out, sample + ".premirna")
            bam_fn = _filter_seqs(bam_fn)
            if args.miraligner:
                _cmd_miraligner(bam_fn, out_file, args.sps, args.hairpin, args.out)
                reads = _read_miraligner(out_file)
                out_files.append(out_file)
        else:
            raise ValueError("Format not recognized.")

        if args.miraligner:
            _mirtop(out_files, args.hairpin, args.gtf, args.sps, args.out)

        if not args.miraligner:
            reads = _annotate(reads, matures, precursors)

        out_file = op.join(args.out, sample + ".mirna")
        out_file, dt, dt_pre = _tab_output(reads, out_file, sample)
        try:
            vcf_file = op.join(args.out, sample + ".vcf")
            if not file_exists(vcf_file):
                # if True:
                create_vcf(dt_pre, matures, gtf, vcf_file)
            try:
                import vcf
                vcf.Reader(filename=vcf_file)
            except Exception as e:
                logger.warning(e.__doc__)
                logger.warning(e.message)
        except Exception as e:
            # traceback.print_exc()
            logger.warning(e.__doc__)
            logger.warning(e.message)
        if isinstance(dt, pd.DataFrame):
            out_dts.append(dt)

    if out_dts:
        _create_counts(out_dts, args.out)
    else:
        print("No files analyzed!")


Instruction:
Given the following Python 3 function, write the documentation
def chdir(new_dir):
    cur_dir = os.getcwd()
    _mkdir(new_dir)
    os.chdir(new_dir)
    try:
        yield
    finally:
        os.chdir(cur_dir)

Response: Context manager to temporarily change to a new directory.


Instruction:
Given the following Python 3 function, write the documentation
def _get_flavor():
    target = op.join("seqcluster", "flavor")
    url = "https://github.com/lpantano/seqcluster.git"
    if not os.path.exists(target):
    #   shutil.rmtree("seqcluster")
        subprocess.check_call(["git", "clone","-b", "flavor", "--single-branch", url])
    return op.abspath(target)

Response: Download flavor from github


Instruction:
Create a Python 3 function to
install bcbio to the local_install folder

Response:
def _install(path, args):
    """
    small helper for installation in case outside bcbio
    """
    try:
       from bcbio import install as bcb
    except:
        raise ImportError("It needs bcbio to do the quick installation.")

    path_flavor = _get_flavor()
    s = {"fabricrc_overrides": {"system_install": path,
                                "local_install": os.path.join(path, "local_install"),
                                "use_sudo": "false",
                                "edition": "minimal"}}
    s = {"flavor": path_flavor,
         # "target": "[brew, conda]",
         "vm_provider": "novm",
         "hostname": "localhost",
         "fabricrc_overrides": {"edition": "minimal",
                                "use_sudo": "false",
                                "keep_isolated": "true",
                                "conda_cmd": bcb._get_conda_bin(),
                                "distribution": "__auto__",
                                "dist_name": "__auto__"}}

    s["actions"] = ["install_biolinux"]
    s["fabricrc_overrides"]["system_install"] = path
    s["fabricrc_overrides"]["local_install"] = os.path.join(path, "local_install")
    cbl = bcb.get_cloudbiolinux(bcb.REMOTES)
    sys.path.insert(0, cbl["dir"])
    cbl_deploy = __import__("cloudbio.deploy", fromlist=["deploy"])
    cbl_deploy.deploy(s)


Instruction:
Write a Python 3 script for
installing required genome data files in place.

Response:
def _install_data(data_dir, path_flavor, args):
    """Upgrade required genome data files in place.
    """
    try:
       from bcbio import install as bcb
    except:
        raise ImportError("It needs bcbio to do the quick installation.")

    bio_data = op.join(path_flavor, "../biodata.yaml")
    s = {"flavor": path_flavor,
         # "target": "[brew, conda]",
         "vm_provider": "novm",
         "hostname": "localhost",
         "fabricrc_overrides": {"edition": "minimal",
                                "use_sudo": "false",
                                "keep_isolated": "true",
                                "conda_cmd": bcb._get_conda_bin(),
                                "distribution": "__auto__",
                                "dist_name": "__auto__"}}
    s["actions"] = ["setup_biodata"]
    s["fabricrc_overrides"]["data_files"] = data_dir
    s["fabricrc_overrides"]["galaxy_home"] = os.path.join(data_dir, "galaxy")
    cbl = bcb.get_cloudbiolinux(bcb.REMOTES)
    s["genomes"] = _get_biodata(bio_data, args)
    sys.path.insert(0, cbl["dir"])
    cbl_deploy = __import__("cloudbio.deploy", fromlist=["deploy"])
    cbl_deploy.deploy(s)


Instruction:
Can you tell what is the following Python 3 function doing
def predictions(args):

    logger.info(args)
    logger.info("reading sequeces")
    out_file = os.path.abspath(os.path.splitext(args.json)[0] + "_prediction.json")
    data = load_data(args.json)
    out_dir = os.path.abspath(safe_dirs(os.path.join(args.out, "predictions")))

    logger.info("make predictions")
    data = is_tRNA(data, out_dir, args)

    if args.coral:
        logger.info("make CoRaL predictions")
        run_coral(data, out_dir, args)
    write_data(data[0], out_file)
    logger.info("Done")

Response: Create predictions of clusters


Instruction:
Can you create a Python 3 function that
sorts loci according to number of sequences mapped there.

Response:
def sort_precursor(c, loci):
    """
    Sort loci according to number of sequences mapped there.
    """
    # Original Py 2.7 code
    #data_loci = map(lambda (x): [x, loci[x].chr, int(loci[x].start), int(loci[x].end), loci[x].strand, len(c.loci2seq[x])], c.loci2seq.keys())
    # 2to3 suggested Py 3 rewrite
    data_loci = [[x, loci[x].chr, int(loci[x].start), int(loci[x].end), loci[x].strand, len(c.loci2seq[x])] for x in list(c.loci2seq.keys())]
    data_loci = sorted(data_loci, key=itemgetter(5), reverse=True)
    return data_loci


Instruction:
Can you create a Python 3 function that
selects the best precursor asuming size around 100 nt

Response:
def best_precursor(clus, loci):
    """
    Select best precursor asuming size around 100 nt
    """
    data_loci = sort_precursor(clus, loci)
    current_size = data_loci[0][5]
    best = 0
    for item, locus in enumerate(data_loci):
        if locus[3] - locus[2] > 70:
            if locus[5] > current_size * 0.8:
                best = item
                break
    best_loci = data_loci[best]
    del data_loci[best]
    data_loci.insert(0, best_loci)
    return data_loci


Instruction:
Write a Python 3 script to
use bedtools to intersect coordinates

Response:
def select_snps(mirna, snp, out):
    """
    Use bedtools to intersect coordinates
    """
    with open(out, 'w') as out_handle:
        print(_create_header(mirna, snp, out), file=out_handle, end="")
        snp_in_mirna = pybedtools.BedTool(snp).intersect(pybedtools.BedTool(mirna), wo=True)
        for single in snp_in_mirna:
            if single[10] == "miRNA" and len(single[3]) + len(single[4]) == 2:
                line = []
                rel_p = _lift_positions(single)
                line.append(_get_mirna_name(single[16]))
                line.append(str(rel_p))
                line.append(single[2])
                line.append(_complement(single[3], single[14]))
                line.append(_complement(single[4], single[14]))
                line.append(single[5])
                line.append(single[6])
                line.append(single[7])
                print("\t".join(line), file=out_handle, end="")
    return out


Instruction:
Here you have a function in Python 3, explain what it does
def up_threshold(x, s, p):
    if 1.0 * x/s >= p:
        return True
    elif stat.binom_test(x, s, p) > 0.01:
        return True
    return False

Response: function to decide if similarity is below cutoff


Instruction:
Can you write a function in Python 3 where it
gets the region inside the vector with more expression

Response:
def _scan(positions):
    """get the region inside the vector with more expression"""
    scores = []
    for start in range(0, len(positions) - 17, 5):
        end = start = 17
        scores.add(_enrichment(positions[start:end], positions[:start], positions[end:]))


Instruction:
Create a Python 3 function to
create clusters of sequences in a single file.

Response:
def cluster(args):
    """
    Creating clusters
    """

    args = _check_args(args)
    read_stats_file = op.join(args.dir_out, "read_stats.tsv")
    if file_exists(read_stats_file):
        os.remove(read_stats_file)

    bam_file, seq_obj = _clean_alignment(args)

    logger.info("Parsing matrix file")
    seqL, y, l = parse_ma_file(seq_obj, args.ffile)
    # y, l = _total_counts(seqL.keys(), seqL)
    logger.info("counts after: %s" % sum(y.values()))
    logger.info("# sequences after: %s" % l)
    dt = pd.DataFrame({'sample': y.keys(), 'counts': y.values()})
    dt['step'] = 'aligned'
    dt.to_csv(read_stats_file, sep="\t", index=False, header=False, mode='a')

    if len(seqL.keys()) < 10:
        logger.error("It seems you have low coverage. Please check your fastq files have enough sequences.")
        raise ValueError("So few sequences.")

    logger.info("Cleaning bam file")
    y, l = _total_counts(seqL.keys(), seqL)
    logger.info("counts after: %s" % sum(y.values()))
    logger.info("# sequences after: %s" % l)
    dt = pd.DataFrame({'sample': y.keys(), 'counts': y.values()})
    dt['step'] = 'cleaned'
    dt.to_csv(read_stats_file, sep="\t", index=False, header=False, mode='a')

    clusL = _create_clusters(seqL, bam_file, args)
    y, l = _total_counts(clusL.seq.keys(), clusL.seq, aligned=True)
    logger.info("counts after: %s" % sum(y.values()))
    logger.info("# sequences after: %s" % l)
    dt = pd.DataFrame({'sample': y.keys(), 'counts': y.values()})
    dt['step'] = 'clusters'
    dt.to_csv(read_stats_file, sep="\t", index=False, header=False, mode='a')

    logger.info("Solving multi-mapping events in the network of clusters")
    clusLred = _cleaning(clusL, args.dir_out)
    y, l = _total_counts(clusLred.clus, seqL)
    logger.info("counts after: %s" % sum(y.values()))
    logger.info("# sequences after: %s" % l)
    dt = pd.DataFrame({'sample': y.keys(), 'counts': y.values()})
    dt['step'] = 'meta-cluster'
    dt.to_csv(read_stats_file, sep="\t", index=False, header=False, mode='a')
    logger.info("Clusters up to %s" % (len(clusLred.clus.keys())))

    if args.show:
        logger.info("Creating sequences alignment to precursor")
        clusLred = show_seq(clusLred, args.index)

    clusLred = peak_calling(clusLred)

    clusLred = _annotate(args, clusLred)

    logger.info("Creating json and count matrix")
    json_file = _create_json(clusLred, args)

    logger.info("Output file in: %s" % args.dir_out)

    if args.db:
        name = args.db + ".db"
        logger.info("Create database: database/" + name)
        data = load_data(json_file)
        out_dir = op.join(args.dir_out, "database")
        make_database(data, name, out_dir)
    logger.info("Finished")


Instruction:
Explain what the following Python 3 code does
def _check_args(args):
    logger.info("Checking parameters and files")
    args.dir_out = args.out
    args.samplename = "pro"
    global decision_cluster
    global similar
    if not os.path.isdir(args.out):
        logger.warning("the output folder doens't exists")
        os.mkdirs(args.out)
    if args.bed and args.gtf:
        logger.error("cannot provide -b and -g at the same time")
        raise SyntaxError
    if args.debug:
        logger.info("DEBUG messages will be showed in file.")
    if args.bed:
        args.list_files = args.bed
        args.type_ann = "bed"
    if args.gtf:
        args.list_files = args.gtf
        args.type_ann = "gtf"
    logger.info("Output dir will be: %s" % args.dir_out)
    if not all([file_exists(args.ffile), file_exists(args.afile)]):
        logger.error("I/O error: Seqs.ma or Seqs.bam. ")
        raise IOError("Seqs.ma or/and Seqs.bam doesn't exists.")
    if hasattr(args, 'list_files'):
        beds = args.list_files.split(",")
        for filebed in beds:
            if not file_exists(filebed):
                logger.error("I/O error: {0}".format(filebed))
                raise IOError("%s  annotation files doesn't exist" % filebed)
    param.decision_cluster = args.method
    if args.similar:
        param.similar = float(args.similar)
    if args.min_seqs:
        param.min_seqs = int(args.min_seqs)
    return args

Response: check arguments before starting analysis.


Instruction:
Create a Python 3 function to
count total seqs after each step

Response:
def _total_counts(seqs, seqL, aligned=False):
    """
    Counts total seqs after each step
    """
    total = Counter()
    if isinstance(seqs, list):
        if not aligned:
            l = len([total.update(seqL[s].freq) for s in seqs])
        else:
            l = len([total.update(seqL[s].freq) for s in seqs if seqL[s].align > 0])
    elif isinstance(seqs, dict):
        [total.update(seqs[s].get_freq(seqL)) for s in seqs]
        l = sum(len(seqs[s].idmembers) for s in seqs)
    return total, l


Instruction:
Here you have a function in Python 3, explain what it does
def _get_annotation(c, loci):
    data_ann_temp = {}
    data_ann = []
    counts = Counter()
    for lid in c.loci2seq:
        # original Py 2.7 code
        #for dbi in loci[lid].db_ann.keys():
        #    data_ann_temp[dbi] = {dbi: map(lambda (x): loci[lid].db_ann[dbi].ann[x].name, loci[lid].db_ann[dbi].ann.keys())}
        # suggestion by 2to3
        for dbi in list(loci[lid].db_ann.keys()):
            data_ann_temp[dbi] = {dbi: [loci[lid].db_ann[dbi].ann[x].name for x in list(loci[lid].db_ann[dbi].ann.keys())]}
            logger.debug("_json_: data_ann_temp %s %s" % (dbi, data_ann_temp[dbi]))
            counts[dbi] += 1
        # original Py 2.7 code
        #data_ann = data_ann + map(lambda (x): data_ann_temp[x], data_ann_temp.keys())
        # suggestion by 2to3
        data_ann = data_ann + [data_ann_temp[x] for x in list(data_ann_temp.keys())]
        logger.debug("_json_: data_ann %s" % data_ann)
    counts = {k: v for k, v in counts.iteritems()}
    total_loci = sum([counts[db] for db in counts])
    valid_ann = [k for k, v in counts.iteritems() if up_threshold(v, total_loci, 0.7)]
    return data_ann, valid_ann

Response: get annotation of transcriptional units


Instruction:
How would you code a function in Python 3 to
sum the metacluster by samples.

Response:
def _sum_by_samples(seqs_freq, samples_order):
    """
    Sum sequences of a metacluster by samples.
    """
    n = len(seqs_freq[seqs_freq.keys()[0]].freq.keys())
    y = np.array([0] * n)
    for s in seqs_freq:
        x = seqs_freq[s].freq
        exp = [seqs_freq[s].freq[sam] for sam in samples_order]
        y = list(np.array(exp) + y)
    return y


Instruction:
How would you explain what the following Python 3 function does
def _annotate(args, setclus):
    logger.info("Creating bed file")
    bedfile = generate_position_bed(setclus)
    a = pybedtools.BedTool(bedfile, from_string=True)
    beds = []
    logger.info("Annotating clusters")
    if hasattr(args, 'list_files'):
        beds = args.list_files.split(",")
        for filebed in beds:
            logger.info("Using %s " % filebed)
            db = os.path.basename(filebed)
            b = pybedtools.BedTool(filebed)
            c = a.intersect(b, wo=True)
            setclus = anncluster(c, setclus, db, args.type_ann, args.feature_id)
    return setclus

Response:
annotate transcriptional units with
    gtf file provided by - b option


Instruction:
Write a Python 3 function for
cleaning alignment and detect complexity.

Response:
def _clean_alignment(args):
    """
    Prepare alignment for cluster detection.
    """
    logger.info("Clean bam file with highly repetitive reads with low counts. sum(counts)/n_hits > 1%")
    bam_file, seq_obj = clean_bam_file(args.afile, args.mask)
    logger.info("Using %s file" % bam_file)
    detect_complexity(bam_file, args.ref, args.out)
    return bam_file, seq_obj


Instruction:
Make a summary of the following Python 3 code
def _create_clusters(seqL, bam_file, args):
    clus_obj = []
    cluster_file = op.join(args.out, "cluster.bed")
    if not os.path.exists(op.join(args.out, 'list_obj.pk')):
        if not file_exists(cluster_file):
            logger.info("Parsing aligned file")
            logger.info("Merging sequences")
            bedtools = os.path.join(os.path.dirname(sys.executable), "bedtools")
            bedtools = bedtools if os.path.exists(bedtools) else "bedtools"
            parse_cmd = "awk '{i=i+1;print $1\"\\t\"$2\"\\t\"$3\"\\t\"$4\"\\t\"i\"\\t\"$6}'"
            cmd = "{bedtools} bamtobed -i {bam_file} | {parse_cmd} | {bedtools} cluster -s -d 20 -i - > {cluster_file}"
            do.run(cmd.format(**locals()))
        c = pybedtools.BedTool(cluster_file)
        logger.info("Creating clusters")
        clus_obj = detect_clusters(c, seqL, args.min_seqs, args.non_un_gl)
        with open(op.join(args.out, 'list_obj.pk'), 'wb') as output:
            pickle.dump(clus_obj, output, pickle.HIGHEST_PROTOCOL)
    else:
        logger.info("Loading previous clusters")
        with open(op.join(args.out, 'list_obj.pk'), 'rb') as input:
            clus_obj = pickle.load(input)
    # bedfile = pybedtools.BedTool(generate_position_bed(clus_obj), from_string=True)
    # seqs_2_loci = bedfile.intersect(pybedtools.BedTool(aligned_bed, from_string=True), wo=True, s=True)
    # seqs_2_position = add_seqs_position_to_loci(seqs_2_loci, seqL)
    logger.info("%s clusters found" % (len(clus_obj.clusid)))
    return clus_obj

Response: Create clusters for the given sequence L and bam file.


Instruction:
Write a Python 3 function that can
load saved cluster and jump to next step

Response:
def _cleaning(clusL, path):
    """
    Load saved cluster and jump to next step
    """
    backup = op.join(path, "list_obj_red.pk")
    if not op.exists(backup):
        clus_obj = reduceloci(clusL, path)
        with open(backup, 'wb') as output:
            pickle.dump(clus_obj, output, pickle.HIGHEST_PROTOCOL)
        return clus_obj
    else:
        logger.info("Loading previous reduced clusters")
        with open(backup, 'rb') as in_handle:
            clus_obj = pickle.load(in_handle)
        return clus_obj


Instruction:
Can you generate the documentation for the following Python 3 function
def explore(args):
    logger.info("reading sequeces")
    data = load_data(args.json)
    logger.info("get sequences from json")
    #get_sequences_from_cluster()
    c1, c2 = args.names.split(",")
    seqs, names = get_sequences_from_cluster(c1, c2, data[0])
    loci = get_precursors_from_cluster(c1, c2, data[0])
    logger.info("map all sequences to all loci")
    print("%s" % (loci))
    map_to_precursors(seqs, names, loci, os.path.join(args.out, "map.tsv"), args)
    #map_sequences_w_bowtie(sequences, precursors)
    logger.info("plot sequences on loci")
    #get_matrix_position()
    #plot_sequences()
    logger.info("Done")

Response: Create mapping of sequences of two clusters


Instruction:
Can you generate the documentation for the following Python 3 function
def prepare(args):
    try:
        f = open(args.config, 'r')
        seq_out = open(op.join(args.out, "seqs.fastq"), 'w')
        ma_out = open(op.join(args.out, "seqs.ma"), 'w')
    except IOError as e:
        traceback.print_exc()
        raise IOError("Can not create output files: %s, %s or read %s" % (op.join(args.out, "seqs.ma"), op.join(args.out, "seqs.fastq"), args.config))
    logger.info("Reading sequeces")
    seq_l, sample_l = _read_fastq_files(f, args)
    logger.info("Creating matrix with unique sequences")
    logger.info("Filtering: min counts %s, min size %s, max size %s, min shared %s" % (args.minc, args.minl, args.maxl, args.min_shared))
    _create_matrix_uniq_seq(sample_l, seq_l, ma_out, seq_out, args.min_shared)
    logger.info("Finish preprocessing. Get a sorted BAM file of seqs.fa and run seqcluster cluster.")

Response: Read all seq. fa files and create a matrix and unique fasta files.


Instruction:
Here you have a function in Python 3, explain what it does
def _read_fasta_files(f, args):
    seq_l = {}
    sample_l = []
    idx = 1
    for line1 in f:
        line1 = line1.strip()
        cols = line1.split("\t")
        with open(cols[0], 'r') as fasta:
            sample_l.append(cols[1])
            for line in fasta:
                if line.startswith(">"):
                    idx += 1
                    counts = int(re.search("x([0-9]+)", line.strip()).group(1))
                else:
                    seq = line.strip()
                    seq = seq[0:int(args.maxl)] if len(seq) > int(args.maxl) else seq
                    if counts > int(args.minc) and len(seq) > int(args.minl):
                        if seq not in seq_l:
                            seq_l[seq] = sequence_unique(idx, seq)
                        seq_l[seq].add_exp(cols[1], counts)
    return seq_l, sample_l

Response: read fasta files of each sample and generate a seq_obj for each unique sequence in each sample


Instruction:
Given the following Python 3 function, write the documentation
def _read_fastq_files(f, args):
    seq_l = {}
    sample_l = []
    idx = 1
    p = re.compile("^[ATCGNU]+$")
    with open(op.join(args.out, "stats_prepare.tsv"), 'w') as out_handle:
        for line1 in f:
            line1 = line1.strip()
            cols = line1.split("\t")
            # if not is_fastq(cols[0]):
            #    raise ValueError("file is not fastq: %s" % cols[0])
            with open_fastq(cols[0]) as handle:
                sample_l.append(cols[1])
                total = added = 0
                for line in handle:
                    if line.startswith("@") or line.startswith(">"):
                        seq = handle.next().strip()
                        if not p.match(seq):
                            continue
                        idx += 1
                        total += 1
                        keep = {}
                        counts = int(re.search("x([0-9]+)", line.strip()).group(1))
                        if is_fastq(cols[0]):
                            handle.next().strip()
                            qual = handle.next().strip()
                        else:
                            qual = "I" * len(seq)
                        qual = qual[0:int(args.maxl)] if len(qual) > int(args.maxl) else qual
                        seq = seq[0:int(args.maxl)] if len(seq) > int(args.maxl) else seq
                        if counts > int(args.minc) and len(seq) > int(args.minl):
                            added += 1
                            if seq in keep:
                                keep[seq].update(qual)
                            else:
                                keep[seq] = quality(qual)
                            if seq not in seq_l:
                                seq_l[seq] = sequence_unique(idx, seq)
                            seq_l[seq].add_exp(cols[1], counts)
                            seq_l[seq].quality = keep[seq].get()
                print("total\t%s\t%s" % (idx, cols[1]), file=out_handle, end="")
                print("added\t%s\t%s" % (len(seq_l), cols[1]), file=out_handle, end="")
                logger.info("%s: Total read %s ; Total added %s" % (cols[1], idx, len(seq_l)))
    return seq_l, sample_l

Response: read fasta files of each sample and generate a seq_obj for each unique sequence in each sample


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _create_matrix_uniq_seq(sample_l, seq_l, maout, out, min_shared):
    skip = 0
    if int(min_shared) > len(sample_l):
        min_shared = len(sample_l)
    maout.write("id\tseq")
    for g in sample_l:
        maout.write("\t%s" % g)
    for s in seq_l.keys():
        seen = sum([1 for g in seq_l[s].group if seq_l[s].group[g] > 0])
        if seen < int(min_shared):
            skip += 1
            continue
        maout.write("\nseq_%s\t%s" % (seq_l[s].idx, seq_l[s].seq))
        for g in sample_l:
            if g in seq_l[s].group:
                maout.write("\t%s" % seq_l[s].group[g])
            else:
                maout.write("\t0")
        qual = "".join(seq_l[s].quality)
        out.write("@seq_%s\n%s\n+\n%s\n" % (seq_l[s].idx, seq_l[s].seq, qual))
    out.close()
    maout.close()
    logger.info("Total skipped due to --min-shared parameter (%s) : %s" % (min_shared, skip))

Response: create matrix counts for each different sequence in all the fasta files


Instruction:
Create a Python 3 function to
run some CoRaL modules to predict small RNA

Response:
def run_coral(clus_obj, out_dir, args):
    """
    Run some CoRaL modules to predict small RNA function
    """
    if not args.bed:
        raise ValueError("This module needs the bed file output from cluster subcmd.")
    workdir = op.abspath(op.join(args.out, 'coral'))
    safe_dirs(workdir)
    bam_in = op.abspath(args.bam)
    bed_in = op.abspath(args.bed)
    reference = op.abspath(args.ref)
    with chdir(workdir):
        bam_clean = coral.prepare_bam(bam_in, bed_in)
        out_dir = op.join(workdir, "regions")
        safe_dirs(out_dir)
        prefix = "seqcluster"
        loci_file = coral.detect_regions(bam_clean, bed_in, out_dir, prefix)
        coral.create_features(bam_clean, loci_file, reference, out_dir)


Instruction:
Given the following Python 3 function, write the documentation
def is_tRNA(clus_obj, out_dir, args):
    ref = os.path.abspath(args.reference)
    utils.safe_dirs(out_dir)
    for nc in clus_obj[0]:
        c = clus_obj[0][nc]
        loci = c['loci']
        out_fa = "cluster_" + nc
        if loci[0][3] - loci[0][2] < 500:
            with make_temp_directory() as tmpdir:
                os.chdir(tmpdir)
                get_loci_fasta({loci[0][0]: [loci[0][0:5]]}, out_fa, ref)
                summary_file, str_file = _run_tRNA_scan(out_fa)
                if "predictions" not in c:
                    c['predictions'] = {}
                c['predictions']['tRNA'] = _read_tRNA_scan(summary_file)
                score = _read_tRNA_scan(summary_file)
                logger.debug(score)
                shutil.move(summary_file, op.join(out_dir, summary_file))
                shutil.move(str_file, op.join(out_dir, str_file))
        else:
            c['errors'].add("precursor too long")
        clus_obj[0][nc] = c

    return clus_obj

Response: Checks if each cluster precursors have a tRNA type.


Instruction:
Here you have a function in Python 3, explain what it does
def _read_tRNA_scan(summary_file):
    score = 0
    if os.path.getsize(summary_file) == 0:
        return 0
    with open(summary_file) as in_handle:
        # header = in_handle.next().strip().split()
        for line in in_handle:
            if not line.startswith("--"):
                pre = line.strip().split()
                score = pre[-1]
    return score

Response: Parse output from tRNA_Scan


Instruction:
Can you generate the documentation for the following Python 3 function
def _run_tRNA_scan(fasta_file):
    out_file = fasta_file + "_trnascan"
    se_file = fasta_file + "_second_str"
    cmd = "tRNAscan-SE -q -o {out_file} -f {se_file} {fasta_file}"
    run(cmd.format(**locals()))
    return out_file, se_file

Response: Run tRNA - scan - SE to predict tRNA


Instruction:
Can you tell what is the following Python 3 function doing
def _parse_mut(mut):
    multiplier = 1
    if mut.startswith("-"):
        mut = mut[1:]
        multiplier = -1
    nt = mut.strip('0123456789')
    pos = int(mut[:-2]) * multiplier
    return nt, pos

Response: Parse mutation field to get position and nts.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _get_reference_position(isomir):
    mut = isomir.split(":")[1]
    if mut == "0":
        return mut
    nt, pos = _parse_mut(mut)
    trim5 = isomir.split(":")[-2]
    off = -1 * len(trim5)
    if trim5.islower():
        off = len(trim5)
    if trim5 == "NA" or trim5 == "0":
        off = 0
    # print(isomir)
    # print([mut, pos, off, nt])
    return "%s%s" % (pos + off, nt)

Response: Get the reference position of the isomir.


Instruction:
Implement a function in Python 3 to
get pct of variants respect to the reference using reads and different sequences

Response:
def _get_pct(isomirs, mirna):
    """
    Get pct of variants respect to the reference
    using reads and different sequences
    """
    pass_pos = []
    for isomir in isomirs.iterrows():
        mir = isomir[1]["chrom"]
        mut = isomir[1]["sv"]
        mut_counts = isomir[1]["counts"]
        total = mirna.loc[mir, "counts"] * 1.0 - mut_counts
        mut_diff = isomir[1]["diff"]
        ratio = mut_counts / total
        if mut_counts > 10 and ratio  > 0.4 and mut != "0" and mut_diff > 1:
            isomir[1]["ratio"] = ratio
            pass_pos.append(isomir[1])
    return pass_pos


Instruction:
How would you explain what the following Python 3 function does
def _print_header(data):
    print("##fileformat=VCFv4.2", file=STDOUT, end="")
    print("##source=seqbuster2.3", file=STDOUT, end="")
    print("##reference=mirbase", file=STDOUT, end="")
    for pos in data:
        print("##contig=<ID=%s>" % pos["chrom"], file=STDOUT, end="")
    print('##INFO=<ID=ID,Number=1,Type=String,Description="miRNA name">', file=STDOUT, end="")
    print('##FORMAT=<ID=GT,Number=1,Type=Integer,Description="Genotype">', file=STDOUT, end="")
    print('##FORMAT=<ID=NR,Number=A,Type=Integer,Description="Total reads supporting the variant">', file=STDOUT, end="")
    print('##FORMAT=<ID=NS,Number=A,Type=Float,Description="Total number of different sequences supporting the variant">', file=STDOUT, end="")
    print("#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tSAMP001", file=STDOUT, end="")

Response: Print the header of the sequence buster2. 3 vcf file.


Instruction:
Create a Python 3 function for
printing vcf line following rules.

Response:
def print_vcf(data):
    """Print vcf line following rules."""
    id_name = "."
    qual = "."
    chrom = data['chrom']
    pos = data['pre_pos']
    nt_ref = data['nt'][1]
    nt_snp = data['nt'][0]
    flt = "PASS"
    info = "ID=%s" % data['mature']
    frmt = "GT:NR:NS"
    gntp = "%s:%s:%s" % (_genotype(data), data["counts"], data["diff"])
    print("\t".join(map(str, [chrom, pos, id_name, nt_ref, nt_snp, qual, flt, info, frmt, gntp])), file=STDOUT, end="")


Instruction:
Create a Python 3 function for
making position at precursor scale

Response:
def liftover(pass_pos, matures):
    """Make position at precursor scale"""
    fixed_pos = []
    _print_header(pass_pos)
    for pos in pass_pos:
        mir = pos["mature"]
        db_pos = matures[pos["chrom"]]
        mut = _parse_mut(pos["sv"])
        print([db_pos[mir], mut, pos["sv"]])
        pos['pre_pos'] = db_pos[mir][0] + mut[1] - 1
        pos['nt'] = list(mut[0])
        fixed_pos.append(pos)
        print_vcf(pos)
    return fixed_pos


Instruction:
Implement a function in Python 3 to
create a vcf file of changes for all samples.

Response:
def create_vcf(isomirs, matures, gtf, vcf_file=None):
    """
    Create vcf file of changes for all samples.
    PASS will be ones with > 3 isomiRs supporting the position
         and > 30% of reads, otherwise LOW
    """
    global STDOUT
    isomirs['sv'] = [_get_reference_position(m) for m in isomirs["isomir"]]
    mirna = isomirs.groupby(['chrom']).sum()
    sv = isomirs.groupby(['chrom', 'mature', 'sv'], as_index=False).sum()
    sv["diff"] = isomirs.groupby(['chrom', 'mature', 'sv'], as_index=False).size().reset_index().loc[:,0]
    pass_pos = _get_pct(sv, mirna)
    if vcf_file:
        with open(vcf_file, 'w') as out_handle:
            STDOUT = out_handle
            pass_pos = liftover(pass_pos, matures)

    if gtf:
        vcf_genome_file = vcf_file.replace(".vcf", "_genome.vcf")
        with open(vcf_genome_file, 'w') as out_handle:
            STDOUT = out_handle
            pass_pos = liftover_to_genome(pass_pos, gtf)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _get_seqs_from_cluster(seqs, seen):
    already_in = set()
    not_in = []

    already_in = map(seen.get, seqs)
    # if isinstance(already_in, list):
    already_in = filter(None, already_in)
    not_in = set(seqs) - set(seen.keys())
    # for s in seqs:
    #    if s in seen:
    #        already_in.add(seen[s])
    #    else:
    #        not_in.append(s)
    return list(set(already_in)), list(not_in)

Response: Returns the sequences that are already part of the cluster


Instruction:
Write a Python 3 script to
reduce number of loci a cluster has

Response:
def reduceloci(clus_obj,  path):
    """reduce number of loci a cluster has
    :param clus_obj: cluster object object
    :param path: output path
    """
    filtered = {}
    n_cluster = 0
    large = 0
    current = clus_obj.clusid
    logger.info("Number of loci: %s" % len(clus_obj.loci.keys()))
    bar = ProgressBar(maxval=len(current))
    bar.start()
    bar.update(0)
    for itern, idmc in enumerate(current):
        bar.update(itern)
        logger.debug("_reduceloci: cluster %s" % idmc)
        c = copy.deepcopy(list(current[idmc]))

        n_loci = len(c)
        if n_loci < 1000:
            filtered, n_cluster = _iter_loci(c, clus_obj.clus, (clus_obj.loci, clus_obj.seq), filtered, n_cluster)
        else:
            large += 1
            n_cluster += 1
            _write_cluster(c, clus_obj.clus, clus_obj.loci, n_cluster, path)
            filtered[n_cluster] = _add_complete_cluster(n_cluster, c, clus_obj.clus)
    clus_obj.clus = filtered

    seqs = 0
    for idc in filtered:
        seqs += len(filtered[idc].idmembers)
    logger.info("seqs in clusters %s" % (seqs))

    logger.info("Clusters too long to be analized: %s" % large)
    logger.info("Number of clusters removed because low number of reads: %s" % REMOVED)
    logger.info("Number of clusters with conflicts: %s" % CONFLICT)
    return clus_obj


Instruction:
How would you implement a function in Python 3 that
writes all the loci for complex meta - clusters to a single file.

Response:
def _write_cluster(metacluster, cluster, loci, idx, path):
    """
    For complex meta-clusters, write all the loci for further debug
    """
    out_file = op.join(path, 'log', str(idx) + '.bed')
    with utils.safe_run(out_file):
        with open(out_file, 'w') as out_handle:
            for idc in metacluster:
                for idl in cluster[idc].loci2seq:
                    pos = loci[idl].list()
                    print("\t".join(pos[:4] + [str(len(cluster[idc].loci2seq[idl]))] + [pos[-1]]), file=out_handle, end="")


Instruction:
Given the following Python 3 function, write the documentation
def _iter_loci(meta, clusters, s2p, filtered, n_cluster):
    global CONFLICT
    loci = dict(zip(meta, [clusters[idc] for idc in meta]))

    n_loci = len(meta)
    n_loci_prev = n_loci + 1
    cicle = 0
    # [logger.note("BEFORE %s %s %s" % (c.id, idl, len(c.loci2seq[idl]))) for idl in c.loci2seq]
    internal_cluster = {}
    if n_loci == 1:
        n_cluster += 1
        filtered[n_cluster] = clusters[meta[0]]
        filtered[n_cluster].update(id=n_cluster)
        filtered[n_cluster].set_freq(s2p[1])

    while n_loci < n_loci_prev and n_loci != 1:
        n_loci_prev = n_loci
        cicle += 1
        if (cicle % 1) == 0:
            logger.debug("_iter_loci:number of cicle: %s with n_loci %s" % (cicle, n_loci))
        loci_similarity = _calculate_similarity(loci)
        internal_cluster = _merge_similar(loci, loci_similarity)
        n_loci = len(internal_cluster)
        loci = internal_cluster
        logger.debug("_iter_loci: n_loci %s" % n_loci)

    if n_loci > 1:
        n_internal_cluster = sorted(internal_cluster.keys(), reverse=True)[0]
        CONFLICT += 1
        internal_cluster = _solve_conflict(internal_cluster, s2p, n_internal_cluster)

    internal_cluster = _clean_cluster(internal_cluster)

    for idc in internal_cluster:
        n_cluster += 1
        logger.debug("_iter_loci: add to filtered %s" % n_cluster)
        filtered[n_cluster] = internal_cluster[idc]
        filtered[n_cluster].id = n_cluster
        filtered[n_cluster].update(id=n_cluster)
        filtered[n_cluster].set_freq(s2p[1])
    logger.debug("_iter_loci: filtered %s" % filtered.keys())

    # for new_c in internal_cluster.values():
    #    [logger.note("%s %s %s %s" % (meta, new_c.id, idl, len(new_c.loci2seq[idl]))) for idl in new_c.loci2seq]
    return filtered, n_cluster

Response: Iterate through all loci and decide if they are part of the same TU or not.


Instruction:
Write a Python 3 script to
return 1 cluster per loci

Response:
def _convert_to_clusters(c):
    """Return 1 cluster per loci"""
    new_dict = {}
    n_cluster = 0
    logger.debug("_convert_to_cluster: loci %s" % c.loci2seq.keys())
    for idl in c.loci2seq:
        n_cluster += 1
        new_c = cluster(n_cluster)
        #new_c.id_prev = c.id
        new_c.loci2seq[idl] = c.loci2seq[idl]
        new_dict[n_cluster] = new_c
    logger.debug("_convert_to_cluster: new ids %s" % new_dict.keys())
    return new_dict


Instruction:
Can you tell what is the following Python 3 function doing
def _calculate_similarity(c):
    ma = {}
    for idc in c:
        set1 = _get_seqs(c[idc])
        [ma.update({(idc, idc2): _common(set1, _get_seqs(c[idc2]), idc, idc2)}) for idc2 in c if idc != idc2 and (idc2, idc) not in ma]
    # logger.debug("_calculate_similarity_ %s" % ma)
    return ma

Response: Calculate a similarity matrix of % of shared sequence


Instruction:
Write a Python 3 script for
getting all sequences in a cluster knowing loci

Response:
def _get_seqs(list_idl):
    """get all sequences in a cluster knowing loci"""
    seqs = set()
    for idl in list_idl.loci2seq:
        # logger.debug("_get_seqs_: loci %s" % idl)
        [seqs.add(s) for s in list_idl.loci2seq[idl]]
    # logger.debug("_get_seqs_: %s" % len(seqs))
    return seqs


Instruction:
Can you write a function in Python 3 where it
calculates the common percentage of sequences

Response:
def _common(s1, s2, i1, i2):
    """calculate the common % percentage of sequences"""
    c = len(set(s1).intersection(s2))
    t = min(len(s1), len(s2))
    pct = 1.0 * c / t * t
    is_gt = up_threshold(pct, t * 1.0, parameters.similar)
    logger.debug("_common: pct %s of clusters:%s %s = %s" % (1.0 * c / t, i1, i2, is_gt))
    if pct < parameters.similar and is_gt and pct > 0:
        pct = parameters.similar
    return pct / t


Instruction:
Here you have a function in Python 3, explain what it does
def _is_consistent(pairs, common, clus_seen, loci_similarity):
    all_true1 = all([all([common and loci_similarity[(p, c)] > parameters.similar  for p in pairs if (p, c) in loci_similarity]) for c in clus_seen])
    all_true2 = all([all([common and loci_similarity[(c, p)] > parameters.similar  for p in pairs if (c, p) in loci_similarity]) for c in clus_seen])
    return all_true1 * all_true2

Response:
Check if loci shared that match sequences with all
    clusters seen until now.


Instruction:
Given the following Python 3 function, write the documentation
def _merge_similar(loci, loci_similarity):
    n_cluster = 0
    internal_cluster = {}
    clus_seen = {}
    loci_sorted = sorted(loci_similarity.iteritems(), key=operator.itemgetter(1), reverse=True)
    for pairs, sim in loci_sorted:
        common = sim > parameters.similar
        n_cluster += 1
        logger.debug("_merge_similar:try new cluster %s" % n_cluster)
        new_c = cluster(n_cluster)
        p_seen, p_unseen = [], []
        size = min(len(_get_seqs(loci[pairs[0]])), len(_get_seqs(loci[pairs[1]])))
        if common:
            consistent = _is_consistent(pairs, common, clus_seen, loci_similarity)
            logger.debug("_merge_similar: clusters seen: %s" % clus_seen)
            logger.debug("_merge_similar: id %s common %s|%s total  %s consistent %s" % (pairs, sim, common, size, consistent))
            if not consistent:
                continue
            if pairs[0] in clus_seen:
                p_seen.append(pairs[0])
                p_unseen.append(pairs[1])
            if pairs[1] in clus_seen:
                p_seen.append(pairs[1])
                p_unseen.append(pairs[0])
            if len(p_seen) == 0:
                new_c = _merge_cluster(loci[pairs[0]], new_c)
                new_c = _merge_cluster(loci[pairs[1]], new_c)
                [clus_seen.update({p: n_cluster}) for p in pairs]
                internal_cluster[n_cluster] = new_c
            if len(p_seen) == 1:
                idc_seen = clus_seen[p_seen[0]]
                internal_cluster[idc_seen] = _merge_cluster(loci[p_unseen[0]], internal_cluster[idc_seen])
                clus_seen[p_unseen[0]] = idc_seen
        else:
            logger.debug("_merge_similar: id %s %s  are different" % pairs)
            continue
    internal_cluster.update(_add_unseen(loci, clus_seen, n_cluster))
    logger.debug("_merge_similar: total clus %s" %
                 len(internal_cluster.keys()))
    return internal_cluster

Response: Internal function to reduce loci complexity


Instruction:
How would you explain what the following Python 3 function does
def _merge_cluster(old, new):
    logger.debug("_merge_cluster: %s to %s" % (old.id, new.id))
    logger.debug("_merge_cluster: add idls %s" % old.loci2seq.keys())
    for idl in old.loci2seq:
        # if idl in new.loci2seq:
        #    new.loci2seq[idl] = list(set(new.loci2seq[idl] + old.loci2seq[idl]))
        # new.loci2seq[idl] = old.loci2seq[idl]
        new.add_id_member(old.loci2seq[idl], idl)
    return new

Response: merge one cluster to another


Instruction:
How would you code a function in Python 3 to
return a new set of cluster names that are not in the same cluster.

Response:
def _solve_conflict(list_c, s2p, n_cluster):
    """
    Make sure sequences are counts once.
    Resolve by most-vote or exclussion

    :params list_c: dict of objects cluster
    :param s2p: dict of [loci].coverage = # num of seqs
    :param n_cluster: number of clusters

    return dict: new set of clusters
    """
    logger.debug("_solve_conflict: count once")
    if parameters.decision_cluster == "bayes":
        return decide_by_bayes(list_c, s2p)
    loci_similarity = _calculate_similarity(list_c)
    loci_similarity = sorted(loci_similarity.iteritems(), key=operator.itemgetter(1), reverse=True)
    common = sum([score for p, score in loci_similarity])
    while common > 0:
        n_cluster += 1
        logger.debug("_solve_conflict: ma %s" % loci_similarity)
        pairs = loci_similarity[0][0]
        score = loci_similarity[0][1]
        logger.debug("_solve_conflict: common %s, new %s" % (score, n_cluster))
        if parameters.decision_cluster.startswith("most-voted"):
            list_c = _split_cluster_by_most_vote(list_c, pairs)
        else:
            list_c = _split_cluster(list_c, pairs, n_cluster)
        list_c = {k: v for k, v in list_c.iteritems() if len(v.loci2seq) > 0}
        loci_similarity = _calculate_similarity(list_c)
        loci_similarity = sorted(loci_similarity.iteritems(), key=operator.itemgetter(1), reverse=True)
        #logger.note("%s %s" % (pairs, loci_similarity[0][1]))
        common = sum([score for p, score in loci_similarity])
        logger.debug("_solve_conflict: solved clusters %s" % len(list_c.keys()))
    return list_c


Instruction:
Can you create a Python 3 function that
splits cluster by exclussion

Response:
def _split_cluster(c, pairs, n):
    """split cluster by exclussion"""
    old = c[p[0]]
    new = c[p[1]]
    new_c = cluster(n)
    common = set(_get_seqs(old)).intersection(_get_seqs(new))
    for idl in old.loci2seq:
        in_common = list(set(common).intersection(old.loci2seq[idl]))
        if len(in_common) > 0:
            logger.debug("_split_cluster: in_common %s with pair 1" % (len(in_common)))
            new_c.add_id_member(in_common, idl)
            old.loci2seq[idl] = list(set(old.loci2seq[idl]) - set(common))
            logger.debug("_split_cluster: len old %s with pair 1" % (len(old.loci2seq)))
    for idl in new.loci2seq:
        in_common = list(set(common).intersection(new.loci2seq[idl]))
        if len(in_common) > 0:
            logger.debug("_split_cluster: in_common %s with pair 2" % (len(in_common)))
            new_c.add_id_member(in_common, idl)
            new.loci2seq[idl] = list(set(new.loci2seq[idl]) - set(common))
            logger.debug("_split_cluster: len old %s with pair 2" % (len(new.loci2seq)))
    old.update()
    new.update()
    old.loci2seq = {k: v for k, v in old.loci2seq.iteritems() if len(v) > 0}
    new.loci2seq = {k: v for k, v in new.loci2seq.iteritems() if len(v) > 0}
    c[n] = new
    c[p[0]] = old
    c[p[1]] = new
    return c


Instruction:
Write a Python 3 function that can
split cluster by most - vote strategy

Response:
def _split_cluster_by_most_vote(c, p):
    """split cluster by most-vote strategy"""
    old, new = c[p[0]], c[p[1]]
    old_size = _get_seqs(old)
    new_size = _get_seqs(new)
    logger.debug("_most_vote: size of %s with %s - %s with %s" % (old.id, len(old_size), new.id, len(new_size)))
    if len(old_size) > len(new_size):
        keep, remove = old, new
    else:
        keep, remove = new, old
    common = list(set(old_size).intersection(new_size))
    logger.debug("_most_vote: keep %s remove  %s with common %s" % (keep.id, remove.id, len(common)))
    for idl in remove.loci2seq:
        if len(common) > 0:
            remove.loci2seq[idl] = list(set(remove.loci2seq[idl]) - set(common))
    keep.loci2seq = {k: v for k, v in keep.loci2seq.iteritems() if len(v) > 0}
    remove.loci2seq = {k: v for k, v in remove.loci2seq.iteritems() if len(v) > 0}
    keep.update()
    remove.update()
    c[keep.id] = keep
    c[remove.id] = remove
    return c


Instruction:
Explain what the following Python 3 code does
def _clean_cluster(list_c):
    global REMOVED
    init = len(list_c)
    list_c = {k: v for k, v in list_c.iteritems() if len(_get_seqs(v)) > parameters.min_seqs}
    logger.debug("_clean_cluster: number of clusters %s " % len(list_c.keys()))
    list_c = {k: _select_loci(v) for k, v in list_c.iteritems()}
    end = len(list_c)
    REMOVED += init - end
    return list_c

Response:
Remove clusters with less than 10 sequences and
    loci with size smaller than 60%


Instruction:
How would you explain what the following Python 3 function does
def _select_loci(c):
    loci_len = {k: len(v) for k, v in c.loci2seq.iteritems()}
    logger.debug("_select_loci: number of loci %s" % len(c.loci2seq.keys()))
    loci_len_sort = sorted(loci_len.iteritems(), key=operator.itemgetter(1), reverse=True)
    max_size = loci_len_sort[0][1]
    logger.debug("_select_loci: max size %s" % max_size)
    loci_clean = {locus: c.loci2seq[locus] for locus, size in loci_len_sort if size > 0.8 * max_size}
    c.loci2seq = loci_clean
    removed = list(set(c.idmembers.keys()) - set(_get_seqs(c)))
    c.add_id_member(removed, loci_len_sort[0][0])
    logger.debug("_select_loci: number of loci %s after cleaning" % len(c.loci2seq.keys()))
    return c

Response: Select only loci with most abundant sequences


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _solve_loci_deprecated(c, locilen_sorted, seen_seqs, filtered, maxseq, n_cluster):
    first_run = 0
    seen_seqs = list()
    n_cluster += 1
    logger.debug("_solve_loci:new cluster %s" % n_cluster)
    new_c = cluster(n_cluster)
    for idl, lenl in locilen_sorted:
        locus_seqs = c.loci2seq[idl]
        if first_run == 0:
            seen_seqs = locus_seqs
            first_run = 1
            first_idl = idl
        intersect = list(set(seen_seqs).intersection(locus_seqs))
        common = 0
        if intersect:
            common = len(intersect)*1.0/min(len(seen_seqs), len(locus_seqs))
        logger.debug("_sole_loci:id %s idl %s len %s max %s seen %s inter %s common %s " % (c.id, idl, lenl, maxseq, len(seen_seqs), len(intersect), common))
        if common*1.0 >= 0.6:
            if lenl*1.0 >= 0.6*maxseq:
                c, new_c, seen_seqs = _merge_loci_in_cluster(c, new_c, idl, seen_seqs)
            else:
                c, new_c, seen_seqs = _merge_with_first_loci(c, new_c, first_idl, idl, seen_seqs)
        else:
            c = _remove_seqs_from_loci(c, idl, seen_seqs)
    filtered[n_cluster] = new_c
    return c, seen_seqs, filtered, n_cluster

Response: deprecated function to reduce loci complexity


Instruction:
Implement a Python 3 function for
parsing annotation to get nice description

Response:
def _get_description(string):
    """
    Parse annotation to get nice description
    """
    ann = set()
    if not string:
        return "This cluster is inter-genic."
    for item in string:
        for db in item:
            ann = ann.union(set(item[db]))
    return "annotated as: %s ..." % ",".join(list(ann)[:3])


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _set_format(profile):
    x = set()
    for sample in profile:
        x = x.union(set(profile[sample].keys()))
    if not x:
        return ''
    end, start = max(x), min(x)
    x = range(start, end, 4)
    scaled_profile = defaultdict(list)
    for pos in x:
        for sample in profile:
            y = _get_closer(profile[sample], pos)
            if y:
                scaled_profile[sample].append(profile[sample][y])
            else:
                scaled_profile[sample].append(0)
    return {'x': list(x), 'y': scaled_profile, 'names': scaled_profile.keys()}

Response: Prepare dict to list of y values with same x


Instruction:
Here you have a function in Python 3, explain what it does
def _insert_data(con, data):
    with con:
        cur = con.cursor()
        cur.execute("DROP TABLE IF EXISTS clusters;")
        cur.execute("CREATE TABLE clusters(Id INT, Description TEXT, Locus TEXT, Annotation TEXT, Sequences TEXT, Profile TXT, Precursor TXT)")
        for c in data[0]:
            locus = json.dumps(data[0][c]['loci'])
            annotation = json.dumps(data[0][c]['ann'])
            description = _get_description(data[0][c]['ann'])
            sequences = json.dumps(_get_sequences(data[0][c]))
            keys = data[0][c]['freq'][0].values()[0].keys()
            profile = "Not available."
            if 'profile' in data[0][c]:
                profile = json.dumps(_set_format(data[0][c]['profile']))
            precursor = json.dumps(data[0][c].get('precursor'))
            cur.execute("INSERT INTO clusters VALUES(%s, '%s', '%s', '%s', '%s', '%s', '%s')" % (c, description, locus, annotation, sequences, profile, precursor))

Response: insert data into the database


Instruction:
Can you tell what is the following Python 3 function doing
def parse_align_file(file_in):
    loc_id = 1
    bedfile_clusters = ""
    bamfile = pybedtools.BedTool(file_in)
    bed = pybedtools.BedTool.bam_to_bed(bamfile)
    for c, start, end, name, q, strand in bed:
        loc_id += 1
        bedfile_clusters += "%s\t%s\t%s\t%s\t%s\t%s\n" % \
                            (c, start, end, name, loc_id, strand)
    return bedfile_clusters

Response: Parse sam files with aligned sequences


Instruction:
Can you tell what is the following Python 3 function doing
def parse_ma_file(seq_obj, in_file):
    name = ""
    index = 1
    total = defaultdict(int)
    with open(in_file) as handle_in:
        line = handle_in.readline().strip()
        cols = line.split("\t")
        samples = cols[2:]
        for line in handle_in:
            line = line.strip()
            cols = line.split("\t")
            name = int(cols[0].replace("seq_", ""))
            seq = cols[1]
            exp = {}
            for i in range(len(samples)):
                exp[samples[i]] = int(cols[i+2])
                total[samples[i]] += int(cols[i+2])
            index = index+1
            if name in seq_obj:
                seq_obj[name].set_freq(exp)
                seq_obj[name].set_seq(seq)
            # new_s = sequence(seq, exp, index)
            # seq_l[name] = new_s
    seq_obj = _normalize_seqs(seq_obj, total)
    return seq_obj, total, index

Response: read seqs. ma file and create dict with sequence object


Instruction:
How would you explain what the following Python 3 function does
def read_gtf_line(cols, field="name"):
    field = field.lower()
    try:
        group = cols[2]
        attrs = cols[8].split(";")
        name = [attr.strip().split(" ")[1] for attr in attrs if attr.strip().split(" ")[0].lower().endswith(field)]
        if not name:
            name = [attr.strip().split(" ")[1] for attr in attrs if attr.strip().split(" ")[0].lower().endswith("gene_id")]
        if not name:
            name = ["None"]
        biotype = [attr.strip().split(" ")[1] for attr in attrs if attr.strip().split(" ")[0].lower().endswith("biotype")]
        if biotype:
            group = biotype[0]
        c = cols[0]
        s = int(cols[3])
        e = int(cols[4])
        st = cols[6]
        return [c, s, e, st, group, name[0]]
    except(Exception, e):
        logger.error(cols)
        logger.error("File is not in correct format")
        logger.error("Expect chr source feature start end . strand attributes")
        logger.error("Attributes are 'gene_name SNCA; gene_id ENSG; '")
        logger.error("The 3rd column is used as type of small RNA (like miRNA)")
        logger.error("at least should contains '; *name NAME; '")
        logger.error(e)
        raise

Response: parse a gtf line to get class and name information


Instruction:
Given the following Python 3 function, write the documentation
def _position_in_feature(pos_a, pos_b):
    strd = "-"
    if pos_a[2] in pos_b[2]:
        strd = "+"
    if pos_a[2] in "+" and pos_b[2] in "+":
        lento5 = pos_a[0] - pos_b[1] + 1
        lento3 = pos_a[1] - pos_b[1] + 1
    if pos_a[2] in "+" and pos_b[2] in "-":
        lento5 = pos_a[1] - pos_b[0] + 1
        lento3 = pos_a[0] - pos_b[1] + 1
    if pos_a[2] in "-" and pos_b[2] in "+":
        lento5 = pos_a[0] - pos_b[1] + 1
        lento3 = pos_a[1] - pos_b[0] + 1
    if pos_a[2] in "-" and pos_b[2] in "-":
        lento3 = pos_a[0] - pos_b[0] + 1
        lento5 = pos_a[1] - pos_b[1] + 1
    else:
        lento5 = pos_a[0] - pos_b[0] + 1
        lento3 = pos_a[1] - pos_b[1] + 1
    return lento5, lento3, strd

Response: return distance to 3 and 5 end of the feature


Instruction:
Write a Python 3 script to
intersect transcription position with annotation files

Response:
def anncluster(c, clus_obj, db, type_ann, feature_id="name"):
    """intersect transcription position with annotation files"""
    id_sa, id_ea, id_id, id_idl, id_sta = 1, 2, 3, 4, 5
    if type_ann == "bed":
        id_sb = 7
        id_eb = 8
        id_stb = 11
        id_tag = 9
    ida = 0
    clus_id = clus_obj.clus
    loci_id = clus_obj.loci
    db = os.path.splitext(db)[0]
    logger.debug("Type:%s\n" % type_ann)
    for cols in c.features():
        if type_ann == "gtf":
            cb, sb, eb, stb, db, tag = read_gtf_line(cols[6:], feature_id)
        else:
            sb = int(cols[id_sb])
            eb = int(cols[id_eb])
            stb = cols[id_stb]
            tag = cols[id_tag]
        id = int(cols[id_id])
        idl = int(cols[id_idl])
        if (id in clus_id):
            clus = clus_id[id]
            sa = int(cols[id_sa])
            ea = int(cols[id_ea])
            ida += 1
            lento5, lento3, strd = _position_in_feature([sa, ea, cols[id_sta]], [sb, eb, stb])
            if db in loci_id[idl].db_ann:
                ann = annotation(db, tag, strd, lento5, lento3)
                tdb = loci_id[idl].db_ann[db]
                tdb.add_db_ann(ida, ann)
                loci_id[idl].add_db(db, tdb)
            else:
                ann = annotation(db, tag, strd, lento5, lento3)
                tdb = dbannotation(1)
                tdb.add_db_ann(ida, ann)
                loci_id[idl].add_db(db, tdb)
            clus_id[id] = clus
    clus_obj.clus = clus_id
    clus_obj.loci = loci_id
    return clus_obj


Instruction:
How would you explain what the following Python 3 function does
def detect_complexity(bam_in, genome, out):
    if not genome:
        logger.info("No genome given. skipping.")
        return None
    out_file = op.join(out, op.basename(bam_in) + "_cov.tsv")
    if file_exists(out_file):
        return None
    fai = genome + ".fai"
    cov = pybedtools.BedTool(bam_in).genome_coverage(g=fai, max=1)
    cov.saveas(out_file)
    total = 0
    for region in cov:
        if region[0] == "genome" and int(region[1]) != 0:
            total += float(region[4])
    logger.info("Total genome with sequences: %s " % total)

Response: Detect complexity of small RNA sequences


Instruction:
Here you have a function in Python 3, explain what it does
def clean_bam_file(bam_in, mask=None):
    seq_obj = defaultdict(int)
    if mask:
        mask_file = op.splitext(bam_in)[0] + "_mask.bam"
        if not file_exists(mask_file):
            pybedtools.BedTool(bam_file).intersect(b=mask, v=True).saveas(mask_file)
        bam_in = mask_file
    out_file = op.splitext(bam_in)[0] + "_rmlw.bam"
    # bam.index(bam_in, {'algorithm':{}})
    run("samtools index %s" % bam_in)
    if not file_exists(bam_in + ".bai"):
        raise IOError("Failed to created bam index of %s. Try to do it manually" % bam_in)
    bam_handle = pysam.AlignmentFile(bam_in, "rb")
    with pysam.AlignmentFile(out_file, "wb", template=bam_handle) as out_handle:
        for read in bam_handle.fetch():
            seq_name = int(read.query_name.replace('seq_', ''))
            match_size = [nts for oper, nts in read.cigartuples if oper == 0]
            subs_size = [nts for oper, nts in read.cigartuples if oper == 4]
            if match_size[0] < 17:
                continue
            if subs_size:
                if subs_size[0] > 3:
                    continue
            try:
                nh = read.get_tag('NH')
            except KeyError:
                nh = 1
            seq_obj[seq_name] = sequence(seq_name)
            seq_obj[seq_name].align = nh
            out_handle.write(read)
    return out_file, seq_obj

Response: Remove from alignment reads with low counts and highly of hits from the bam file.