Instruction:
Can you write a function in Python 3 where it
parses the merge file of sequences position to create clusters that will have all sequences that shared any position on the genome.

Response:
def detect_clusters(c, current_seq, MIN_SEQ, non_un_gl=False):
    """
    Parse the merge file of sequences position to create clusters that will have all
    sequences that shared any position on the genome

    :param c: file from bedtools with merge sequence positions
    :param current_seq: list of sequences
    :param MIN_SEQ: int cutoff to keep the cluster or not. 10 as default

    :return: object with information about:
        * cluster
        * dict with sequences (as keys) and cluster_id (as value)
        * sequences
        * loci

    """
    current_loci = {}
    current_clus = {}
    # sequence2clusters = [set()] * (max(current_seq.keys()) + 2)
    sequence2clusters = defaultdict(set)
    lindex = 0
    eindex = 0
    previous_id = 0
    for line in c.features():
        c, start, end, name, score, strand, c_id = line
        name = int(name.replace('seq_', ''))
        pos = int(start) if strand == "+" else int(end)
        if name not in current_seq:
            continue
        if c.find('Un_gl') > -1 and non_un_gl:
            continue
        if c_id != previous_id:
            if previous_id > 0:
                if len(current_clus[eindex].idmembers) < MIN_SEQ:
                    for s in current_clus[eindex].idmembers:
                        sequence2clusters[s] = sequence2clusters[s] - set([eindex])
                    del current_clus[eindex]

            logger.debug("detect_cluster: %s %s %s" % (c_id, previous_id, name))
            lindex += 1
            eindex += 1
            current_clus[eindex] = cluster(eindex)
            newpos = position(lindex, c, start, end, strand)
            current_loci[lindex] = newpos

        # update locus, sequences in each line
        current_loci[lindex].end = int(end)
        current_loci[lindex].coverage[pos] += 1
        size = range(pos, pos + current_seq[name].len)
        current_loci[lindex].counts.update(dict(zip(size, [current_seq[name].total()] * current_seq[name].len)))
        current_clus[eindex].idmembers[name] = 1
        current_clus[eindex].add_id_member([name], lindex)
        current_seq[name].add_pos(lindex, pos)
        # current_seq[name].align = 1
        previous_id = c_id
        sequence2clusters[name].add(eindex)
    logger.info("%s Clusters read" % eindex)
    # merge cluster with shared sequences
    metacluster_obj, cluster_id = _find_metaclusters(current_clus, sequence2clusters, current_seq, MIN_SEQ)

    return cluster_info_obj(current_clus, metacluster_obj, current_loci, current_seq)


Instruction:
Can you write a function in Python 3 where it
finds all meta - clusters that share sequences

Response:
def _find_metaclusters(clus_obj, sequence2clusters, current_seq, min_seqs):
    """
    Mask under same id all clusters that share sequences
    :param clus_obj: cluster object coming from detect_cluster
    :param min_seqs: int cutoff to keep the cluster or not. 10 as default

    :return: updated clus_obj and dict with seq_id: cluster_id
    """
    seen = defaultdict(int)
    metacluster = defaultdict(set)
    c_index = len(sequence2clusters)
    logger.info("Creating meta-clusters based on shared sequences: %s" % c_index)
    meta_idx = 1
    bar = ProgressBar(maxval=c_index)
    bar.start()
    bar.update()
    for itern, name in enumerate(sequence2clusters):
        clusters = sequence2clusters[name]
        if len(clusters) == 0:
            c_index -= 1
            continue
        current_seq[name].align = 1
        meta_idx += 1
        bar.update(itern)
        already_in = _common(clusters, seen)
        _update(clusters, meta_idx, seen)
        metacluster[meta_idx] = metacluster[meta_idx].union(clusters)

        if already_in:
            for seen_metacluster in already_in:
                clusters2merge = metacluster[seen_metacluster]
                metacluster[meta_idx] = metacluster[meta_idx].union(clusters2merge)
                _update(clusters2merge, meta_idx, seen)
                # metacluster[seen_metacluster] = 0
                del metacluster[seen_metacluster]
    logger.info("%s metaclusters from %s sequences" % (len(metacluster), c_index))

    return metacluster, seen


Instruction:
Here you have a function in Python 3, explain what it does
def _find_families_deprecated(clus_obj, min_seqs):
    logger.info("Creating meta-clusters based on shared sequences.")
    seen = defaultdict()
    metacluster = defaultdict(list)
    c_index = clus_obj.keys()
    meta_idx = 0
    with ProgressBar(maxval=len(c_index), redirect_stdout=True) as p:
        for itern, c in enumerate(c_index):
            p.update(itern)
            clus = clus_obj[c]
            if len(clus.idmembers.keys()) < min_seqs:
                del clus_obj[c]
                continue
            logger.debug("reading cluster %s" % c)
            logger.debug("loci2seq  %s" % clus.loci2seq)
            already_in, not_in = _get_seqs_from_cluster(clus.idmembers.keys(), seen)
            logger.debug("seen %s news %s" % (already_in, not_in))
            meta_idx += 1
            metacluster[meta_idx].append(c)
            seen.update(dict(zip(not_in, [meta_idx] * len(not_in))))
            if len(already_in) > 0:
                logger.debug("seen in %s" % already_in)
                for eindex in already_in:
                    for cluster in metacluster[eindex]:
                        metacluster[meta_idx].append(cluster)
                        prev_clus = clus_obj[cluster]
                        logger.debug("_find_families: prev %s current %s" % (eindex, clus.id))
                        # add current seqs to seen cluster
                        seqs_in = prev_clus.idmembers.keys()
                        seen.update(dict(zip(seqs_in, [meta_idx] * len(seqs_in))))
                        # for s_in_clus in prev_clus.idmembers:
                        #    seen[s_in_clus] = meta_idx
                    #    clus.idmembers[s_in_clus] = 1
                    # add current locus to seen cluster
                    # for loci in prev_clus.loci2seq:
                    #    logger.debug("adding %s" % loci)
                        # if not loci_old in current_clus[eindex].loci2seq:
                    #    clus.add_id_member(list(prev_clus.loci2seq[loci]), loci)
                    # logger.debug("loci %s" % clus.loci2seq.keys())
                    del metacluster[eindex]
                # clus_obj[c] = clus

                # logger.debug("num cluster %s" % len(clus_obj.keys()))
    logger.info("%s clusters merged" % len(metacluster))

    return metacluster, seen

Response: Find all possible families of the cluster in which the cluster is not in the same id.


Instruction:
Here you have a function in Python 3, explain what it does
def peak_calling(clus_obj):
    new_cluster = {}
    for cid in clus_obj.clus:
        cluster = clus_obj.clus[cid]
        cluster.update()
        logger.debug("peak calling for %s" % cid)
        bigger = cluster.locimaxid
        if bigger in clus_obj.loci:
            s, e = min(clus_obj.loci[bigger].counts.keys()), max(clus_obj.loci[bigger].counts.keys())
            scale = s
            if clus_obj.loci[bigger].strand == "-":
                scale = e
            logger.debug("bigger %s at %s-%s" % (bigger, s, e))
            dt = np.array([0] * (abs(e - s) + 12))
            for pos in clus_obj.loci[bigger].counts:
                ss = abs(int(pos) - scale) + 5
                dt[ss] += clus_obj.loci[bigger].counts[pos]
        x = np.array(range(0, len(dt)))
        logger.debug("x %s and y %s" % (x, dt))
        # tab = pd.DataFrame({'x': x, 'y': dt})
        # tab.to_csv( str(cid) + "peaks.csv", mode='w', header=False, index=False)
        if len(x) > 35 + 12:
            peaks = list(np.array(pysen.pysenMMean(x, dt)) - 5)
            logger.debug(peaks)
        else:
            peaks =  ['short']
        cluster.peaks = peaks
        new_cluster[cid] = cluster
    clus_obj.clus = new_cluster
    return clus_obj

Response: Run peak calling inside each cluster


Instruction:
Here you have a function in Python 3, explain what it does
def _generate_reads(seq, name):
    reads = dict()
    if len(seq) < 130 and len(seq) > 70:
        reads.update(_mature(seq[:40], 0, name))
        reads.update(_mature(seq[-40:], len(seq) - 40, name))
        reads.update(_noise(seq, name))
        reads.update(_noise(seq, name, 25))
    return reads

Response: Main function that creates reads from precursors


Instruction:
Create a Python 3 function to
create mature sequences around start and end

Response:
def _mature(subseq, absolute, c,  size=33, total=5000):
    """Create mature sequences around start/end"""
    reads = dict()
    probs = [0.1, 0.2, 0.4, 0.2, 0.1]
    end = 5 + size
    error = [-2, -1, 0, 1, 2]
    for error5 in error:
        for error3 in error:
            s = 5 - error5
            e = end - error3
            seen = subseq[s:e]
            counts = int(probs[error5 + 2] * probs[error3 + 2] * total) + 1
            name = "seq_%s_%s_%s_x%s" % (c, s + absolute, e + absolute, counts)
            reads[name] = (seen, counts)
    return reads


Instruction:
How would you implement a function in Python 3 that
creates mature sequences around start and end

Response:
def _noise(seq, c, size=33, total=1000):
    """Create mature sequences around start/end"""
    reads = dict()
    seen = 0
    while seen < total:
        s = random.randint(0, len(seq) - size)
        e = s + size + random.randint(-5,5)
        p = random.uniform(0, 0.1)
        counts = int(p * total) + 1
        seen += counts
        name = "seq_%s_%s_%s_x%s" % (c, s, e, counts)
        reads[name] = (seq[s:e], counts)
    return reads


Instruction:
Explain what the following Python 3 code does
def stats(args):
    logger.info("Reading sequeces")
    data = parse_ma_file(args.ma)
    logger.info("Get sequences from sam")
    is_align = _read_sam(args.sam)
    is_json, is_db = _read_json(args.json)
    res = _summarise_sam(data, is_align, is_json, is_db)
    _write_suma(res, os.path.join(args.out, "stats_align.dat"))
    logger.info("Done")

Response: Create stats from the analysis


Instruction:
Can you create a Python 3 function that
runs the provided command and return the result.

Response:
def run(cmd, data=None, checks=None, region=None, log_error=True,
        log_stdout=False):
    """Run the provided command, logging details and checking for errors.
    """
    try:
        logger.debug(" ".join(str(x) for x in cmd) if not isinstance(cmd, basestring) else cmd)
        _do_run(cmd, checks, log_stdout)
    except:
        if log_error:
            logger.info("error at command")
        raise


Instruction:
Explain what the following Python 3 code does
def _normalize_cmd_args(cmd):
    if isinstance(cmd, basestring):
        # check for standard or anonymous named pipes
        if cmd.find(" | ") > 0 or cmd.find(">(") or cmd.find("<("):
            return "set -o pipefail; " + cmd, True, find_bash()
        else:
            return cmd, True, None
    else:
        return [str(x) for x in cmd], False, None

Response: Normalize subprocess arguments to handle list commands string and pipes.


Instruction:
How would you implement a function in Python 3 that
performs running and check results.

Response:
def _do_run(cmd, checks, log_stdout=False):
    """Perform running and check results, raising errors for issues.
    """
    cmd, shell_arg, executable_arg = _normalize_cmd_args(cmd)
    s = subprocess.Popen(cmd, shell=shell_arg, executable=executable_arg,
                         stdout=subprocess.PIPE,
                         stderr=subprocess.STDOUT, close_fds=True)
    debug_stdout = collections.deque(maxlen=100)
    while 1:
        line = s.stdout.readline()
        if line:
            debug_stdout.append(line)
            if log_stdout:
                logger.debug(line.rstrip())
            else:
                logger.debug(line.rstrip())
        exitcode = s.poll()
        if exitcode is not None:
            for line in s.stdout:
                debug_stdout.append(line)
            if exitcode is not None and exitcode != 0:
                error_msg = " ".join(cmd) if not isinstance(cmd, basestring) else cmd
                error_msg += "\n"
                error_msg += "".join(debug_stdout)
                s.communicate()
                s.stdout.close()
                raise subprocess.CalledProcessError(exitcode, error_msg)
            else:
                break
    s.communicate()
    s.stdout.close()
    # Check for problems not identified by shell return codes
    if checks:
        for check in checks:
            if not check():
                raise IOError("External command failed")


Instruction:
Create a Python 3 function for
returning dict with sequences = [ cluster1 cluster2... ]

Response:
def _dict_seq_locus(list_c, loci_obj, seq_obj):
    """
    return dict with sequences = [ cluster1, cluster2 ...]
    """
    seqs = defaultdict(set)
    # n = len(list_c.keys())
    for c in list_c.values():
        for l in c.loci2seq:
            [seqs[s].add(c.id) for s in c.loci2seq[l]]

    common = [s for s in seqs if len(seqs[s]) > 1]
    seqs_in_c = defaultdict(float)
    for c in list_c.values():
        for l in c.loci2seq:
            # total = sum([v for v in loci_obj[l].coverage.values()])
            for s in c.loci2seq[l]:
                if s in common:
                    pos = seq_obj[s].pos[l]
                    # cov = 1.0 * loci_obj[l].coverage[pos] / total
                    cov = 1.0 * loci_obj[l].coverage[pos]
                    if seqs_in_c[(s, c.id)] < cov:
                        seqs_in_c[(s, c.id)] = cov
    seqs_in_c = _transform(seqs_in_c)
    return seqs_in_c


Instruction:
Can you create a Python 3 function that
updates the PMF with new data.

Response:
def Update(self, data):
        """Updates the PMF with new data.
        data: string cookie type
        """
        for hypo in self.Values():
            like = self.Likelihood(data, hypo)
            self.Mult(hypo, like)
        self.Normalize()


Instruction:
How would you explain what the following Python 3 function does
def Likelihood(self, data, hypo):
        mix = self.loci[hypo]
        like = mix[data]
        return like

Response: The likelihood of the data under the hypothesis.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def show_seq(clus_obj, index):
    current = clus_obj.clus
    clus_seqt = clus_obj.seq
    clus_locit = clus_obj.loci

    itern = 0
    for idc in current.keys():
        itern += 1
        timestamp = str(idc)
        seqListTemp = ()
        f = open("/tmp/"+timestamp+".fa","w")
        for idl in current[idc].loci2seq.keys():
            seqListTemp = list(set(seqListTemp).union(current[idc].loci2seq[idl]))
        maxscore = 0
        for s in seqListTemp:
            score = calculate_size(clus_seqt[s].freq)
            maxscore = max(maxscore,score)
            clus_seqt[s].score = score
            seq = clus_seqt[s]
            f.write(">"+s+"\n"+seq.seq+"\n")
        f.close()

        locilen_sorted = sorted(current[idc].locilen.iteritems(), key = operator.itemgetter(1),reverse = True)
        lmax = clus_locit[locilen_sorted[0][0]]
        f = open("/tmp/"+timestamp+".bed","w")
        f.write("%s\t%s\t%s\t.\t.\t%s\n" % (lmax.chr,lmax.start,lmax.end,lmax.strand))
        f.close()
        os.system("bedtools  getfasta -s -fi "+index+" -bed /tmp/"+timestamp+".bed -fo /tmp/"+timestamp+".pre.fa")
        os.system("bowtie2-build /tmp/"+timestamp+".pre.fa  /tmp/"+timestamp+".pre.ind >/dev/null 2>&1")
        os.system("bowtie2 --rdg 7,3 --mp 4 --end-to-end --no-head --no-sq  -D 20 -R 3 -N 0 -i S,1,0.8 -L 3 -f /tmp/"+timestamp+".pre.ind /tmp/"+timestamp+".fa -S /tmp/"+timestamp+".map  >>bowtie.log 2>&1")
        f = open("/tmp/"+timestamp+".map","r")
        seqpos = {}
        minv = 10000000
        for line in f:
            line = line.strip()
            cols = line.split("\t")
            seqpos[cols[0]] = int(cols[3])
            if minv>int(cols[3]):
                minv = int(cols[3])
        f.close()
        seqpos_sorted = sorted(seqpos.iteritems(), key = operator.itemgetter(1),reverse = False)
        showseq = ""
        showseq_plain = ""
        for (s,pos) in seqpos_sorted:
            ratio = (clus_seqt[s].score*1.0/maxscore*100.0)
            realScore = (math.log(ratio,2)*2)
            if realScore<0:
                realScore = 0
            # "score %s max %s  ratio %s real %.0f" % (clus_seqt[s].score,maxscore,ratio,realScore)
            ##calculate the mean expression of the sequence and change size letter
            showseq_plain += "
%s%s" % ("".join("." for i in range(pos-1)),realScore+10,s,clus_seqt[s].seq)
            #showseq+ = seqviz.addseq(pos-1,clus_seqt[s].len,clus_seqt[s].seq)
        #current[idc].showseq = showseq
        current[idc].showseq_plain = showseq_plain
        os.system("rm /tmp/"+timestamp+"*")
    clus_obj.clus = current
    clus_obj.seq = clus_seqt
    return clus_obj

Response: Get the precursor and map sequences to it.


Instruction:
Can you write a function in Python 3 where it
prepares BAM file for splitting into smaller cluster positions.

Response:
def prepare_bam(bam_in, precursors):
    """
    Clean BAM file to keep only position inside the bigger cluster
    """
    # use pybedtools to keep valid positions
    # intersect option with -b bigger_cluster_loci
    a = pybedtools.BedTool(bam_in)
    b = pybedtools.BedTool(precursors)
    c = a.intersect(b, u=True)
    out_file = utils.splitext_plus(op.basename(bam_in))[0] + "_clean.bam"
    c.saveas(out_file)
    return op.abspath(out_file)


Instruction:
Given the following Python 3 function, write the documentation
def _reorder_columns(bed_file):
    new_bed = utils.splitext_plus(bed_file)[0] + '_order.bed'
    with open(bed_file) as in_handle:
        with open(new_bed, 'w') as out_handle:
            for line in in_handle:
                cols = line.strip().split("\t")
                cols[3] = _select_anno(cols[3]) + "_" + cols[4]
                cols[4] = "0"
                print("\t".join(cols), file=out_handle, end="")
    return new_bed

Response: Reorder columns to be compatible with CoRaL


Instruction:
Write a Python 3 script for
moving counts to score columns in bed file

Response:
def _fix_score_column(cov_file):
    """
    Move counts to score columns in bed file
    """
    new_cov = utils.splitext_plus(cov_file)[0] + '_fix.cov'
    with open(cov_file) as in_handle:
        with open(new_cov, 'w') as out_handle:
            for line in in_handle:
                cols = line.strip().split("\t")
                cols[4] = cols[6]
                print("\t".join(cols[0:6]), file=out_handle, end="")
    return new_cov


Instruction:
Write a Python 3 function for
detecting regions using first CoRaL module.

Response:
def detect_regions(bam_in, bed_file, out_dir, prefix):
    """
    Detect regions using first CoRaL module
    """
    bed_file = _reorder_columns(bed_file)
    counts_reads_cmd = ("coverageBed -s -counts -b {bam_in} "
                        "-a {bed_file} | sort -k4,4 "
                        "> {out_dir}/loci.cov")
    # with tx_tmpdir() as temp_dir:
    with utils.chdir(out_dir):
        run(counts_reads_cmd.format(min_trimmed_read_len=min_trimmed_read_len, max_trimmed_read_len=max_trimmed_read_len, **locals()), "Run counts_reads")
        loci_file = _fix_score_column(op.join(out_dir, "loci.cov"))
        return loci_file


Instruction:
Create a Python 3 function to
move counts to score columns in bed file Move counts to score columns in bed file

Response:
def _order_antisense_column(cov_file, min_reads):
    """
    Move counts to score columns in bed file
    """
    new_cov = op.join(op.dirname(cov_file), 'feat_antisense.txt')
    with open(cov_file) as in_handle:
        with open(new_cov, 'w') as out_handle:
            print("name\tantisense", file=out_handle, end="")
            for line in in_handle:
                cols = line.strip().split("\t")
                cols[6] = 0 if cols[6] < min_reads else cols[6]
                print("%s\t%s" % (cols[3], cols[6]), file=out_handle, end="")
    return new_cov


Instruction:
Write a Python 3 function for
computing the number of reads per position in a single loci file.

Response:
def _reads_per_position(bam_in, loci_file, out_dir):
    """
    Create input for compute entropy
    """
    data = Counter()
    a = pybedtools.BedTool(bam_in)
    b = pybedtools.BedTool(loci_file)
    c = a.intersect(b, s=True, bed=True, wo=True)
    for line in c:
        end = int(line[1]) + 1 + int(line[2]) if line[5] == "+" else int(line[1]) + 1
        start = int(line[1]) + 1 if line[5] == "+" else int(line[1]) + 1 + int(line[2])
        side5 = "%s\t5p\t%s" % (line[15], start)
        side3 = "%s\t3p\t%s" % (line[15], end)
        data[side5] += 1
        data[side3] += 1

    counts_reads = op.join(out_dir, 'locus_readpos.counts')
    with open(counts_reads, 'w') as out_handle:
        for k in data:
            print(k, file=out_handle, end="")

    return counts_reads


Instruction:
Create a Python 3 function for
creating features from a CoRaL genomic file.

Response:
def create_features(bam_in, loci_file, reference, out_dir):
    """
    Use feature extraction module from CoRaL
    """
    lenvec_plus = op.join(out_dir, 'genomic_lenvec.plus')
    lenvec_minus = op.join(out_dir, 'genomic_lenvec.minus')
    compute_genomic_cmd = ("compute_genomic_lenvectors "
                           "{bam_in} {lenvec_plus} "
                           "{lenvec_minus} "
                           "{min_len} "
                           "{max_len} ")
    index_genomic_cmd = ("index_genomic_lenvectors "
                         "{lenvec} ")
    genomic_lenvec = op.join(out_dir, 'genomic_lenvec')
    feat_len_file = op.join(out_dir, 'feat_lengths.txt')
    compute_locus_cmd = ("compute_locus_lenvectors "
                         "{loci_file} "
                         "{genomic_lenvec} "
                         "{min_len} "
                         "{max_len} "
                         "> {feat_len_file}")
    cov_S_file = op.join(out_dir, 'loci.cov_anti')
    coverage_anti_cmd = ("coverageBed -S -counts -b "
                         "{bam_in} -a {loci_file} "
                         "> {cov_S_file}")
    feat_posentropy = op.join(out_dir, 'feat_posentropy.txt')
    entropy_cmd = ("compute_locus_entropy.rb "
                   "{counts_reads} "
                   "> {feat_posentropy}")
    with utils.chdir(out_dir):
        run(compute_genomic_cmd.format(min_len=min_trimmed_read_len, max_len=max_trimmed_read_len, **locals()), "Run compute_genomic")
        run(index_genomic_cmd.format(lenvec=lenvec_plus), "Run index in plus")
        run(index_genomic_cmd.format(lenvec=lenvec_minus), "Run index in minus")
        run(compute_locus_cmd.format(min_len=min_trimmed_read_len, max_len=max_trimmed_read_len, **locals()), "Run compute locus")
        run(coverage_anti_cmd.format(**locals()), "Run coverage antisense")
        feat_antisense = _order_antisense_column(cov_S_file, min_trimmed_read_len)

        counts_reads = _reads_per_position(bam_in, loci_file, out_dir)
        run(entropy_cmd.format(**locals()), "Run entropy")

        rnafold = calculate_structure(loci_file, reference)


Instruction:
Can you tell what is the following Python 3 function doing
def report(args):
    logger.info("reading sequeces")
    data = load_data(args.json)

    logger.info("create profile")
    data = make_profile(data, os.path.join(args.out, "profiles"), args)
    logger.info("create database")
    make_database(data, "seqcluster.db", args.out)

    logger.info("Done. Download https://github.com/lpantano/seqclusterViz/archive/master.zip to browse the output.")

Response: Create report in html format


Instruction:
Implement a Python 3 function for
returning a list of peaks that are closer than 10

Response:
def _summarize_peaks(peaks):
    """
    merge peaks position if closer than 10
    """
    previous = peaks[0]
    new_peaks = [previous]
    for pos in peaks:
        if pos > previous + 10:
            new_peaks.add(pos)
        previous = pos
    return new_peaks


Instruction:
Can you generate a brief explanation for the following Python 3 code
def find_mature(x, y, win=10):
    previous = min(y)
    peaks = []
    intervals = range(x, y, win)
    for pos in intervals:
        if y[pos] > previous * 10:
            previous = y[pos]
            peaks.add(pos)
    peaks = _summarize_peaks(peaks)

Response: Find mature peaks in the expression profile.


Instruction:
Write a Python 3 script for
collapsing identical sequences and keep Q

Response:
def collapse(in_file):
    """collapse identical sequences and keep Q"""
    keep = Counter()
    with open_fastq(in_file) as handle:
        for line in handle:
            if line.startswith("@"):
                if line.find("UMI") > -1:
                    logger.info("Find UMI tags in read names, collapsing by UMI.")
                    return collapse_umi(in_file)
                seq = handle.next().strip()
                handle.next()
                qual = handle.next().strip()
                if seq in keep:
                    keep[seq].update(qual)
                else:
                    keep[seq] = quality(qual)
    logger.info("Sequences loaded: %s" % len(keep))
    return keep


Instruction:
Make a summary of the following Python 3 code
def collapse_umi(in_file):
    keep = defaultdict(dict)
    with open_fastq(in_file) as handle:
        for line in handle:
            if line.startswith("@"):
                m = re.search('UMI_([ATGC]*)', line.strip())
                umis = m.group(0)
                seq = handle.next().strip()
                handle.next()
                qual = handle.next().strip()
                if (umis, seq) in keep:
                    keep[(umis, seq)][1].update(qual)
                    keep[(umis, seq)][0].update(seq)
                else:
                    keep[(umis, seq)] = [umi(seq), quality(qual)]
    logger.info("Sequences loaded: %s" % len(keep))
    return keep

Response: collapse reads using UMI tags


Instruction:
Implement a Python 3 function for
opening a fastq file using gzip if it is gzipped from bcbio package

Response:
def open_fastq(in_file):
    """ open a fastq file, using gzip if it is gzipped
    from bcbio package
    """
    _, ext = os.path.splitext(in_file)
    if ext == ".gz":
        return gzip.open(in_file, 'rb')
    if ext in [".fastq", ".fq", ".fasta", ".fa"]:
        return open(in_file, 'r')
    return ValueError("File needs to be fastq|fasta|fq|fa [.gz]")


Instruction:
Can you generate a brief explanation for the following Python 3 code
def collapse_fastq(args):
    try:
        umi_fn = args.fastq
        if _is_umi(args.fastq):
            umis = collapse(args.fastq)
            umi_fn = os.path.join(args.out, splitext_plus(os.path.basename(args.fastq))[0] + "_umi_trimmed.fastq")
            write_output(umi_fn, umis, args.minimum)
        seqs = collapse(umi_fn)
        out_file = splitext_plus(os.path.basename(args.fastq))[0] + "_trimmed.fastq"
    except IOError as e:
        logger.error("I/O error({0}): {1}".format(e.errno, e.strerror))
        raise "Can not read file"
    out_file = os.path.join(args.out, out_file)
    write_output(out_file, seqs, args.minimum)
    return out_file

Response: collapse fasq files after adapter trimming


Instruction:
Write a Python 3 function that can
give a doctree remove all non - slide related elements from it.

Response:
def filter_doctree_for_slides(doctree):
    """Given a doctree, remove all non-slide related elements from it."""

    current = 0
    num_children = len(doctree.children)
    while current < num_children:

        child = doctree.children[current]
        child.replace_self(
            child.traverse(no_autoslides_filter)
        )

        if len(doctree.children) == num_children:
            # nothing removed, increment current
            current += 1
        else:
            # a node was removed; retain current and update length
            num_children = len(doctree.children)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _make_title_node(self, node, increment=True):

        parent_title_node = node.parent.next_node(nodes.title)
        nextslide_info = getattr(
            parent_title_node, 'nextslide_info',
            (parent_title_node.deepcopy().children, 1),
        )
        nextslide_info = (
            nextslide_info[0],
            nextslide_info[1] + 1,
        )

        if node.args:
            textnodes, messages = node.state.inline_text(
                node.args[0],
                1,
            )
            new_title = nodes.title(node.args[0], '', *textnodes)

        else:

            title_nodes = nextslide_info[0][:]

            if 'increment' in node.attributes:
                title_nodes.append(
                    nodes.Text(' (%s)' % nextslide_info[1])
                )

            new_title = nodes.title(
                '', '',
                *title_nodes
            )

        new_title.nextslide_info = nextslide_info
        return new_title

Response: Generate a new title node for node.


Instruction:
Can you generate the documentation for the following Python 3 function
def apply(self, builder):

        if 'theme' in self.attributes:
            builder.apply_theme(
                self.attributes['theme'],
                builder.theme_options,
            )

Response: Applies the Slide Configuration to a Builder.


Instruction:
Can you tell what is the following Python 3 function doing
def get_conf(cls, builder, doctree=None):

        # set up the default conf
        result = {
            'theme': builder.config.slide_theme,
            'autoslides': builder.config.autoslides,
            'slide_classes': [],
        }

        # now look for a slideconf node in the doctree and update the conf
        if doctree:
            conf_node = cls.get(doctree)
            if conf_node:
                result.update(conf_node.attributes)

        return result

Response: Return a dictionary of slide configuration for this doctree.


Instruction:
Can you implement a function in Python 3 that
returns a new context dict based on original context.

Response:
def __fix_context(context):
    """Return a new context dict based on original context.

    The new context will be a copy of the original, and some mutable
    members (such as script and css files) will also be copied to
    prevent polluting shared context.
    """

    COPY_LISTS = ('script_files', 'css_files',)

    for attr in COPY_LISTS:
        if attr in context:
            context[attr] = context[attr][:]

    return context


Instruction:
Explain what the following Python 3 code does
def ask_user(d):

    # Print welcome message
    msg = bold('Welcome to the Hieroglyph %s quickstart utility.') % (
        version(),
    )
    print(msg)
    msg = """
This will ask questions for creating a Hieroglyph project, and then ask
some basic Sphinx questions.
"""
    print(msg)

    # set a few defaults that we don't usually care about for Hieroglyph
    d.update({
        'version': datetime.date.today().strftime('%Y.%m.%d'),
        'release': datetime.date.today().strftime('%Y.%m.%d'),
        'make_mode': True,
    })

    if 'project' not in d:
        print('''
The presentation title will be included on the title slide.''')
        sphinx.quickstart.do_prompt(d, 'project', 'Presentation title')
    if 'author' not in d:
        sphinx.quickstart.do_prompt(d, 'author', 'Author name(s)')

    # slide_theme
    theme_entrypoints = pkg_resources.iter_entry_points('hieroglyph.theme')

    themes = [
        t.load()
        for t in theme_entrypoints
    ]

    msg = """
Available themes:

"""

    for theme in themes:
        msg += '\n'.join([
            bold(theme['name']),
            theme['desc'],
            '', '',
        ])

    msg += """Which theme would you like to use?"""
    print(msg)

    sphinx.quickstart.do_prompt(
        d, 'slide_theme', 'Slide Theme', themes[0]['name'],
        sphinx.quickstart.choice(
            *[t['name'] for t in themes]
        ),
    )

    # Ask original questions
    print("")
    sphinx.quickstart.ask_user(d)

Response: Wrap sphinx. quickstart. ask_user and add additional questions.


Instruction:
How would you code a function in Python 3 to
return the context dict for rendering this slide.

Response:
def get_slide_context(self):
        """Return the context dict for rendering this slide."""

        return {
            'title': self.title,
            'level': self.level,
            'content': self.content,
            'classes': self.classes,
            'slide_classes': self._filter_classes(exclude='content-'),
            'content_classes': self._filter_classes(include='content-'),
            'slide_number': self.slide_number,
            'config': self._translator.builder.config,
            'id': self.id,
        }


Instruction:
Can you write a function in Python 3 where it
adds the slide number to the output if enabled.

Response:
def _add_slide_number(self, slide_no):
        """Add the slide number to the output if enabled."""

        if self.builder.config.slide_numbers:
            self.body.append(
                '\n
%s
\n' % (slide_no,),
            )


Instruction:
How would you explain what the following Python 3 function does
def _add_slide_footer(self, slide_no):

        if self.builder.config.slide_footer:
            self.body.append(
                '\n
%s
\n' % (
                    self.builder.config.slide_footer,
                ),
            )

Response: Add the slide footer to the output if enabled.


Instruction:
Can you create a Python 3 function that
inspects the Sphinx configuration and update for slide - linking.

Response:
def inspect_config(app):
    """Inspect the Sphinx configuration and update for slide-linking.

    If links from HTML to slides are enabled, make sure the sidebar
    configuration includes the template and add the necessary theme
    directory as a loader so the sidebar template can be located.

    If the sidebar configuration already includes ``slidelink.html``
    (in any key), the configuration will not be changed. If the
    configuration is not specified, we'll attempt to emulate what
    Sphinx does by default.
    """

    # avoid import cycles :/
    from hieroglyph import writer

    # only reconfigure Sphinx if we're generating HTML
    if app.builder.name not in HTML_BUILDERS:
        return

    if app.config.slide_link_html_to_slides:

        # add the slide theme dir as a Loader
        app.builder.templates.loaders.append(
            SphinxFileSystemLoader(
                os.path.join(
                    os.path.dirname(__file__), 'themes', 'slides',
                )
            )
        )

        # add the "show slides" sidebar template
        if not app.config.html_sidebars:
            # no sidebars explicitly defined, mimic the old style
            # behavior + slide links
            app.config.html_sidebars = {
                '**': [
                    'localtoc.html',
                    'relations.html',
                    'sourcelink.html',
                    SLIDELINK_TEMPLATE,
                    'searchbox.html',
                ],
            }
        else:
            # sidebars defined, add the template if needed
            included = False
            for glob, templates in app.config.html_sidebars:
                if SLIDELINK_TEMPLATE in templates:
                    included = True
                    break

            if not included:
                # the slidelink template was not included; append it
                # to the list of sidebars for all templates
                app.config.html_sidebars.setdefault('**', []).append(
                    SLIDELINK_TEMPLATE,
                )

    if app.config.slide_link_html_sections_to_slides:
        # fix up the HTML Translator
        if sphinx.version_info >= (1, 6, 0):
            override_translator = type(
                'SlideLinkTranslator',
                (app.builder.get_translator_class(), object),
                {
                    'depart_title': writer.depart_title,
                },
            )
            app.set_translator(app.builder, override_translator)
        else:
            app.builder.translator_class = type(
                'SlideLinkTranslator',
                (app.builder.translator_class, object),
                {
                    'depart_title': writer.depart_title,
                },
            )


Instruction:
Here you have a function in Python 3, explain what it does
def slide_path(builder, pagename=None):

    return builder.get_relative_uri(
        pagename or builder.current_docname,
        os.path.join(
            builder.app.config.slide_relative_path,
            pagename or builder.current_docname,
        ))

Response: Calculate the relative path to the Slides for pagename.


Instruction:
How would you explain what the following Python 3 function does
def html_path(builder, pagename=None):

    return builder.get_relative_uri(
        pagename or builder.current_docname,
        os.path.join(
            builder.app.config.slide_html_relative_path,
            pagename or builder.current_docname,
        ))

Response: Calculate the relative path to the Slides for pagename.


Instruction:
Can you generate the documentation for the following Python 3 function
def add_link(app, pagename, templatename, context, doctree):

    # we can only show the slidelink if we can resolve the filename
    context['show_slidelink'] = (
        app.config.slide_link_html_to_slides and
        hasattr(app.builder, 'get_outfilename')
    )

    if context['show_slidelink']:
        context['slide_path'] = slide_path(app.builder, pagename)

Response: Add the slides link to the HTML context.


Instruction:
Can you tell what is the following Python 3 function doing
def apply_theme(self, themename, themeoptions):

        # push the existing values onto the Stack
        self._theme_stack.append(
            (self.theme, self.theme_options)
        )

        theme_factory = HTMLThemeFactory(self.app)
        theme_factory.load_additional_themes(self.get_builtin_theme_dirs() + self.config.slide_theme_path)

        self.theme = theme_factory.create(themename)
        self.theme_options = themeoptions.copy()
        self.templates.init(self, self.theme)
        self.templates.environment.filters['json'] = json.dumps

        if self.theme not in self._additional_themes:
            self._additional_themes.append(self.theme)

Response: This will store the existing theme configuration and apply a new one.


Instruction:
Make a summary of the following Python 3 code
def post_process_images(self, doctree):

        super(AbstractSlideBuilder, self).post_process_images(doctree)

        # figure out where this doctree is in relation to the srcdir
        relative_base = (
            ['..'] *
            doctree.attributes.get('source')[len(self.srcdir) + 1:].count('/')
        )

        for node in doctree.traverse(nodes.image):

            if node.get('candidates') is None:
                node['candidates'] = ('*',)

            # fix up images with absolute paths
            if node['uri'].startswith(self.outdir):
                node['uri'] = '/'.join(
                    relative_base + [
                        node['uri'][len(self.outdir) + 1:]
                    ]
                )

Response: Pick the best candidate for all image URIs.


Instruction:
Can you tell what is the following Python 3 function doing
def parse_metadata(section):
  metadata = {}
  metadata_lines = section.split('\n')
  for line in metadata_lines:
    colon_index = line.find(':')
    if colon_index != -1:
      key = line[:colon_index].strip()
      val = line[colon_index + 1:].strip()
      metadata[key] = val

  return metadata

Response: Given the first part of a slide returns metadata associated with it.


Instruction:
Explain what the following Python 3 code does
def postprocess_html(html, metadata):
  if metadata.get('build_lists') and metadata['build_lists'] == 'true':
    html = html.replace('
', '
')
    html = html.replace('
', '
')
  return html

Response: Returns processed HTML to fit into the slide template format.


Instruction:
Implement a Python 3 function for
augmenting a checker method with an augmentation function.

Response:
def augment_visit(linter, checker_method, augmentation):
    """
    Augmenting a visit enables additional errors to be raised (although that case is
    better served using a new checker) or to suppress all warnings in certain circumstances.

    Augmenting functions should accept a 'chain' function, which runs the checker method
    and possibly any other augmentations, and secondly an Astroid node. "chain()" can be
    called at any point to trigger the continuation of other checks, or not at all to
    prevent any further checking.
    """

    if sys.version_info[0] <= 2:
        checker = get_checker(linter, checker_method.im_class)
    else:
        try:
            checker = get_checker(linter, checker_method.__self__.__class__)
        except AttributeError:
            checker = get_checker(linter, get_class(checker_method.__module__, checker_method.__qualname__))

    old_method = getattr(checker, checker_method.__name__)

    def augment_func(node):
        def chain():
            old_method(node)
        augmentation(chain, node)

    setattr(checker, checker_method.__name__, augment_func)


Instruction:
Given the following Python 3 function, write the documentation
def suppress_message(linter, checker_method, message_id_or_symbol, test_func):
    # At some point, pylint started preferring message symbols to message IDs. However this is not done
    # consistently or uniformly - occasionally there are some message IDs with no matching symbols.
    # We try to work around this here by suppressing both the ID and the symbol, if we can find it.
    # This also gives us compatability with a broader range of pylint versions.

    # Similarly, a commit between version 1.2 and 1.3 changed where the messages are stored - see:
    # https://bitbucket.org/logilab/pylint/commits/0b67f42799bed08aebb47babdc9fb0e761efc4ff#chg-reporters/__init__.py
    # Therefore here, we try the new attribute name, and fall back to the old version for
    # compatability with <=1.2 and >=1.3
    msgs_store = getattr(linter, 'msgs_store', linter)

    def get_message_definitions(message_id_or_symbol):
        if hasattr(msgs_store, 'check_message_id'):
            return [msgs_store.check_message_id(message_id_or_symbol)]
        # pylint 2.0 renamed check_message_id to get_message_definition in:
        # https://github.com/PyCQA/pylint/commit/5ccbf9eaa54c0c302c9180bdfb745566c16e416d
        elif hasattr(msgs_store, 'get_message_definition'):
            return [msgs_store.get_message_definition(message_id_or_symbol)]
        # pylint 2.3.0 renamed get_message_definition to get_message_definitions in:
        # https://github.com/PyCQA/pylint/commit/da67a9da682e51844fbc674229ff6619eb9c816a
        elif hasattr(msgs_store, 'get_message_definitions'):
            return msgs_store.get_message_definitions(message_id_or_symbol)
        else:
            raise ValueError('pylint.utils.MessagesStore does not have a get_message_definition(s) method')

    try:
        pylint_messages = get_message_definitions(message_id_or_symbol)
        symbols = [symbol
                   for pylint_message in pylint_messages
                   for symbol in (pylint_message.msgid, pylint_message.symbol)
                   if symbol is not None]
    except UnknownMessage:
        # This can happen due to mismatches of pylint versions and plugin expectations of available messages
        symbols = [message_id_or_symbol]

    def do_suppress(chain, node):
        with Suppress(linter) as s:
            if test_func(node):
                s.suppress(*symbols)
            chain()
    augment_visit(linter, checker_method, do_suppress)

Response: Wrapper for suppress_message that allows the suppression of a message.


Instruction:
Implement a Python 3 function for
querying DB to see if hash is blacklisted

Response:
def lookup_full_hashes(self, hash_values):
        """Query DB to see if hash is blacklisted"""
        q = '''SELECT threat_type,platform_type,threat_entry_type, expires_at < current_timestamp AS has_expired
                FROM full_hash WHERE value IN ({})
        '''
        output = []
        with self.get_cursor() as dbc:
            placeholders = ','.join(['?'] * len(hash_values))
            dbc.execute(q.format(placeholders), [sqlite3.Binary(hv) for hv in hash_values])
            for h in dbc.fetchall():
                threat_type, platform_type, threat_entry_type, has_expired = h
                threat_list = ThreatList(threat_type, platform_type, threat_entry_type)
                output.append((threat_list, has_expired))
        return output


Instruction:
Here you have a function in Python 3, explain what it does
def lookup_hash_prefix(self, cues):
        q = '''SELECT value, MAX(negative_expires_at < current_timestamp) AS negative_cache_expired
                FROM hash_prefix WHERE cue IN ({}) GROUP BY 1
        '''
        output = []
        with self.get_cursor() as dbc:
            dbc.execute(q.format(','.join(['?'] * len(cues))), [sqlite3.Binary(cue) for cue in cues])
            for h in dbc.fetchall():
                value, negative_cache_expired = h
                output.append((bytes(value), negative_cache_expired))
        return output

Response: Lookup hash prefixes by cue Returns a tuple of the first 4 bytes of hash and the negative cache expired.


Instruction:
Here you have a function in Python 3, explain what it does
def store_full_hash(self, threat_list, hash_value, cache_duration, malware_threat_type):
        log.info('Storing full hash %s to list %s with cache duration %s',
                 to_hex(hash_value), str(threat_list), cache_duration)
        qi = '''INSERT OR IGNORE INTO full_hash
                    (value, threat_type, platform_type, threat_entry_type, malware_threat_type, downloaded_at)
                VALUES
                    (?, ?, ?, ?, ?, current_timestamp)
        '''
        qu = "UPDATE full_hash SET expires_at=datetime(current_timestamp, '+{} SECONDS') \
            WHERE value=? AND threat_type=? AND platform_type=? AND threat_entry_type=?"

        i_parameters = [sqlite3.Binary(hash_value), threat_list.threat_type,
                        threat_list.platform_type, threat_list.threat_entry_type, malware_threat_type]
        u_parameters = [sqlite3.Binary(hash_value), threat_list.threat_type,
                        threat_list.platform_type, threat_list.threat_entry_type]

        with self.get_cursor() as dbc:
            dbc.execute(qi, i_parameters)
            dbc.execute(qu.format(int(cache_duration)), u_parameters)

Response: Store full hash found for the given hash prefix.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def cleanup_full_hashes(self, keep_expired_for=(60 * 60 * 12)):
        q = '''DELETE FROM full_hash WHERE expires_at < datetime(current_timestamp, '-{} SECONDS')
        '''
        log.info('Cleaning up full_hash entries expired more than {} seconds ago.'.format(keep_expired_for))
        with self.get_cursor() as dbc:
            dbc.execute(q.format(int(keep_expired_for)))

Response: Remove long expired full_hash entries.


Instruction:
Here you have a function in Python 3, explain what it does
def get_threat_lists(self):
        q = '''SELECT threat_type,platform_type,threat_entry_type FROM threat_list'''
        output = []
        with self.get_cursor() as dbc:
            dbc.execute(q)
            for h in dbc.fetchall():
                threat_type, platform_type, threat_entry_type = h
                threat_list = ThreatList(threat_type, platform_type, threat_entry_type)
                output.append(threat_list)
        return output

Response: Get a list of known threat lists.


Instruction:
Given the following Python 3 function, write the documentation
def get_client_state(self):
        q = '''SELECT threat_type,platform_type,threat_entry_type,client_state FROM threat_list'''
        output = {}
        with self.get_cursor() as dbc:
            dbc.execute(q)
            for h in dbc.fetchall():
                threat_type, platform_type, threat_entry_type, client_state = h
                threat_list_tuple = (threat_type, platform_type, threat_entry_type)
                output[threat_list_tuple] = client_state
        return output

Response: Get a dict of known threat lists including clientState values.


Instruction:
Write a Python 3 script to
add a threat list entry if it does not exist.

Response:
def add_threat_list(self, threat_list):
        """Add threat list entry if it does not exist."""
        q = '''INSERT OR IGNORE INTO threat_list
                    (threat_type, platform_type, threat_entry_type, timestamp)
                VALUES
                    (?, ?, ?, current_timestamp)
        '''
        params = [threat_list.threat_type, threat_list.platform_type, threat_list.threat_entry_type]
        with self.get_cursor() as dbc:
            dbc.execute(q, params)


Instruction:
Given the following Python 3 function, write the documentation
def delete_threat_list(self, threat_list):
        log.info('Deleting cached threat list "{}"'.format(repr(threat_list)))
        q = '''DELETE FROM threat_list
                    WHERE threat_type=? AND platform_type=? AND threat_entry_type=?
        '''
        params = [threat_list.threat_type, threat_list.platform_type, threat_list.threat_entry_type]
        with self.get_cursor() as dbc:
            dbc.execute(q, params)

Response: Delete the threat list entry.


Instruction:
Here you have a function in Python 3, explain what it does
def hash_prefix_list_checksum(self, threat_list):
        q = '''SELECT value FROM hash_prefix
                WHERE threat_type=? AND platform_type=? AND threat_entry_type=?
                ORDER BY value
        '''
        params = [threat_list.threat_type, threat_list.platform_type, threat_list.threat_entry_type]
        with self.get_cursor() as dbc:
            dbc.execute(q, params)
            all_hashes = b''.join(bytes(h[0]) for h in dbc.fetchall())
            checksum = hashlib.sha256(all_hashes).digest()
        return checksum

Response: Returns SHA256 checksum for alphabetically - sorted concatenated list of hash prefixes


Instruction:
Write a Python 3 script for
removing records matching idices from a lexicographically - sorted local threat list.

Response:
def remove_hash_prefix_indices(self, threat_list, indices):
        """Remove records matching idices from a lexicographically-sorted local threat list."""
        batch_size = 40
        q = '''DELETE FROM hash_prefix
                WHERE threat_type=? AND platform_type=? AND threat_entry_type=? AND value IN ({})
        '''
        prefixes_to_remove = self.get_hash_prefix_values_to_remove(threat_list, indices)
        with self.get_cursor() as dbc:
            for i in range(0, len(prefixes_to_remove), batch_size):
                remove_batch = prefixes_to_remove[i:(i + batch_size)]
                params = [
                    threat_list.threat_type,
                    threat_list.platform_type,
                    threat_list.threat_entry_type
                ] + [sqlite3.Binary(b) for b in remove_batch]
                dbc.execute(q.format(','.join(['?'] * len(remove_batch))), params)


Instruction:
Can you create a Python 3 function that
exports all hash prefix values. Returns a list of known hash prefix values.

Response:
def dump_hash_prefix_values(self):
        """Export all hash prefix values.

        Returns a list of known hash prefix values
        """
        q = '''SELECT distinct value from hash_prefix'''
        output = []
        with self.get_cursor() as dbc:
            dbc.execute(q)
            output = [bytes(r[0]) for r in dbc.fetchall()]
        return output


Instruction:
Can you write a function in Python 3 where it
checks events in the queue on a given Tk instance

Response:
def _check_events(tk):
    """Checks events in the queue on a given Tk instance"""

    used = False
    try:
        # Process all enqueued events, then exit.
        while True:
            try:
                # Get an event request from the queue.
                method, args, kwargs, response_queue = tk.tk._event_queue.get_nowait()
            except queue.Empty:
                # No more events to process.
                break
            else:
                # Call the event with the given arguments, and then return
                # the result back to the caller via the response queue.
                used = True
                if tk.tk._debug >= 2:
                    print('Calling event from main thread:', method.__name__, args, kwargs)
                try:
                    response_queue.put((False, method(*args, **kwargs)))
                except SystemExit:
                    raise  # Raises original SystemExit
                except Exception:
                    # Calling the event caused an exception; return the
                    # exception back to the caller so that it can be raised
                    # in the caller's thread.
                    from sys import exc_info  # Python 2 requirement
                    ex_type, ex_value, ex_tb = exc_info()
                    response_queue.put((True, (ex_type, ex_value, ex_tb)))
    finally:
        # Schedule to check again. If we just processed an event, check
        # immediately; if we didn't, check later.
        if used:
            tk.after_idle(_check_events, tk)
        else:
            tk.after(tk.tk._check_period, _check_events, tk)


Instruction:
Given the following Python 3 function, write the documentation
def update_hash_prefix_cache(self):
        try:
            self.storage.cleanup_full_hashes()
            self.storage.commit()
            self._sync_threat_lists()
            self.storage.commit()
            self._sync_hash_prefix_cache()
        except Exception:
            self.storage.rollback()
            raise

Response: Update locally cached threat lists.


Instruction:
How would you code a function in Python 3 to
download full hashes matching hash_prefixes. Also update cache expiration timestamps.

Response:
def _sync_full_hashes(self, hash_prefixes):
        """Download full hashes matching hash_prefixes.

        Also update cache expiration timestamps.
        """
        client_state = self.storage.get_client_state()
        self.api_client.fair_use_delay()
        fh_response = self.api_client.get_full_hashes(hash_prefixes, client_state)

        # update negative cache for each hash prefix
        # store full hash (insert or update) with positive cache bumped up
        for m in fh_response.get('matches', []):
            threat_list = ThreatList(m['threatType'], m['platformType'], m['threatEntryType'])
            hash_value = b64decode(m['threat']['hash'])
            cache_duration = int(m['cacheDuration'].rstrip('s'))
            malware_threat_type = None
            for metadata in m['threatEntryMetadata'].get('entries', []):
                k = b64decode(metadata['key'])
                v = b64decode(metadata['value'])
                if k == 'malware_threat_type':
                    malware_threat_type = v
            self.storage.store_full_hash(threat_list, hash_value, cache_duration, malware_threat_type)

        negative_cache_duration = int(fh_response['negativeCacheDuration'].rstrip('s'))
        for prefix_value in hash_prefixes:
            self.storage.update_hash_prefix_expiration(prefix_value, negative_cache_duration)


Instruction:
Here you have a function in Python 3, explain what it does
def lookup_url(self, url):
        if type(url) is not str:
            url = url.encode('utf8')
        if not url.strip():
            raise ValueError("Empty input string.")
        url_hashes = URL(url).hashes
        try:
            list_names = self._lookup_hashes(url_hashes)
            self.storage.commit()
        except Exception:
            self.storage.rollback()
            raise
        if list_names:
            return list_names
        return None

Response: Look up specified URL in Safe Browsing threat lists.


Instruction:
Implement a Python 3 function for
looking up the full hash entries in blacklists and return the list of lists it was found in.

Response:
def _lookup_hashes(self, full_hashes):
        """Lookup URL hash in blacklists

        Returns names of lists it was found in.
        """
        full_hashes = list(full_hashes)
        cues = [fh[0:4] for fh in full_hashes]
        result = []
        matching_prefixes = {}
        matching_full_hashes = set()
        is_potential_threat = False
        # First lookup hash prefixes which match full URL hash
        for (hash_prefix, negative_cache_expired) in self.storage.lookup_hash_prefix(cues):
            for full_hash in full_hashes:
                if full_hash.startswith(hash_prefix):
                    is_potential_threat = True
                    # consider hash prefix negative cache as expired if it is expired in at least one threat list
                    matching_prefixes[hash_prefix] = matching_prefixes.get(hash_prefix, False) or negative_cache_expired
                    matching_full_hashes.add(full_hash)
        # if none matches, URL hash is clear
        if not is_potential_threat:
            return []
        # if there is non-expired full hash, URL is blacklisted
        matching_expired_threat_lists = set()
        for threat_list, has_expired in self.storage.lookup_full_hashes(matching_full_hashes):
            if has_expired:
                matching_expired_threat_lists.add(threat_list)
            else:
                result.append(threat_list)
        if result:
            return result

        # If there are no matching expired full hash entries
        # and negative cache is still current for all prefixes, consider it safe
        if len(matching_expired_threat_lists) == 0 and sum(map(int, matching_prefixes.values())) == 0:
            log.info('Negative cache hit.')
            return []

        # Now we can assume that there are expired matching full hash entries and/or
        # cache prefix entries with expired negative cache. Both require full hash sync.
        self._sync_full_hashes(matching_prefixes.keys())
        # Now repeat full hash lookup
        for threat_list, has_expired in self.storage.lookup_full_hashes(matching_full_hashes):
            if not has_expired:
                result.append(threat_list)
        return result


Instruction:
Here you have a function in Python 3, explain what it does
def get_threats_lists(self):
        response = self.service.threatLists().list().execute()
        self.set_wait_duration(response.get('minimumWaitDuration'))
        return response['threatLists']

Response: Retrieve all available threat lists


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_threats_update(self, client_state):
        request_body = {
            "client": {
                "clientId": self.client_id,
                "clientVersion": self.client_version,
            },
            "listUpdateRequests": [],
        }
        for (threat_type, platform_type, threat_entry_type), current_state in client_state.items():
            request_body['listUpdateRequests'].append(
                {
                    "threatType": threat_type,
                    "platformType": platform_type,
                    "threatEntryType": threat_entry_type,
                    "state": current_state,
                    "constraints": {
                        "supportedCompressions": ["RAW"]
                    }
                }
            )
        response = self.service.threatListUpdates().fetch(body=request_body).execute()
        self.set_wait_duration(response.get('minimumWaitDuration'))
        return response['listUpdateResponses']

Response: Fetch hash prefixes update for given threat list.


Instruction:
Can you create a Python 3 function that
finds full hashes matching hash prefixes.

Response:
def get_full_hashes(self, prefixes, client_state):
        """Find full hashes matching hash prefixes.

        client_state is a dict which looks like {(threatType, platformType, threatEntryType): clientState}
        """
        request_body = {
            "client": {
                "clientId": self.client_id,
                "clientVersion": self.client_version,
            },
            "clientStates": [],
            "threatInfo": {
                "threatTypes": [],
                "platformTypes": [],
                "threatEntryTypes": [],
                "threatEntries": [],
            }
        }
        for prefix in prefixes:
            request_body['threatInfo']['threatEntries'].append({"hash": b64encode(prefix).decode()})
        for ((threatType, platformType, threatEntryType), clientState) in client_state.items():
            request_body['clientStates'].append(clientState)
            if threatType not in request_body['threatInfo']['threatTypes']:
                request_body['threatInfo']['threatTypes'].append(threatType)
            if platformType not in request_body['threatInfo']['platformTypes']:
                request_body['threatInfo']['platformTypes'].append(platformType)
            if threatEntryType not in request_body['threatInfo']['threatEntryTypes']:
                request_body['threatInfo']['threatEntryTypes'].append(threatEntryType)
        response = self.service.fullHashes().find(body=request_body).execute()
        self.set_wait_duration(response.get('minimumWaitDuration'))
        return response


Instruction:
Write a Python 3 script for
returning a generator of all possible permutations of the URL in canonical form.

Response:
def hashes(self):
        """Hashes of all possible permutations of the URL in canonical form"""
        for url_variant in self.url_permutations(self.canonical):
            url_hash = self.digest(url_variant)
            yield url_hash


Instruction:
How would you implement a function in Python 3 that
converts URL to its canonical form.

Response:
def canonical(self):
        """Convert URL to its canonical form."""
        def full_unescape(u):
            uu = urllib.unquote(u)
            if uu == u:
                return uu
            else:
                return full_unescape(uu)

        def full_unescape_to_bytes(u):
            uu = urlparse.unquote_to_bytes(u)
            if uu == u:
                return uu
            else:
                return full_unescape_to_bytes(uu)

        def quote(s):
            safe_chars = '!"$&\'()*+,-./:;<=>?@[\\]^_`{|}~'
            return urllib.quote(s, safe=safe_chars)

        url = self.url.strip()
        url = url.replace(b'\n', b'').replace(b'\r', b'').replace(b'\t', b'')
        url = url.split(b'#', 1)[0]
        if url.startswith(b'//'):
            url = b'http:' + url
        if len(url.split(b'://')) <= 1:
            url = b'http://' + url
        # at python3 work with bytes instead of string
        # as URL may contain invalid unicode characters
        if self.__py3 and type(url) is bytes:
            url = quote(full_unescape_to_bytes(url))
        else:
            url = quote(full_unescape(url))
        url_parts = urlparse.urlsplit(url)
        if not url_parts[0]:
            url = 'http://{}'.format(url)
            url_parts = urlparse.urlsplit(url)
        protocol = url_parts.scheme
        if self.__py3:
            host = full_unescape_to_bytes(url_parts.hostname)
            path = full_unescape_to_bytes(url_parts.path)
        else:
            host = full_unescape(url_parts.hostname)
            path = full_unescape(url_parts.path)
        query = url_parts.query
        if not query and '?' not in url:
            query = None
        if not path:
            path = b'/'
        has_trailing_slash = (path[-1:] == b'/')
        path = posixpath.normpath(path).replace(b'//', b'/')
        if has_trailing_slash and path[-1:] != b'/':
            path = path + b'/'
        port = url_parts.port
        host = host.strip(b'.')
        host = re.sub(br'\.+', b'.', host).lower()
        if host.isdigit():
            try:
                host = socket.inet_ntoa(struct.pack("!I", int(host)))
            except Exception:
                pass
        elif host.startswith(b'0x') and b'.' not in host:
            try:
                host = socket.inet_ntoa(struct.pack("!I", int(host, 16)))
            except Exception:
                pass
        quoted_path = quote(path)
        quoted_host = quote(host)
        if port is not None:
            quoted_host = '{}:{}'.format(quoted_host, port)
        canonical_url = '{}://{}{}'.format(protocol, quoted_host, quoted_path)
        if query is not None:
            canonical_url = '{}?{}'.format(canonical_url, query)
        return canonical_url


Instruction:
How would you code a function in Python 3 to
try all permutations of hostname and path which can be applied to blacklisted URLs

Response:
def url_permutations(url):
        """Try all permutations of hostname and path which can be applied

        to blacklisted URLs
        """
        def url_host_permutations(host):
            if re.match(r'\d+\.\d+\.\d+\.\d+', host):
                yield host
                return
            parts = host.split('.')
            l = min(len(parts), 5)
            if l > 4:
                yield host
            for i in range(l - 1):
                yield '.'.join(parts[i - l:])

        def url_path_permutations(path):
            yield path
            query = None
            if '?' in path:
                path, query = path.split('?', 1)
            if query is not None:
                yield path
            path_parts = path.split('/')[0:-1]
            curr_path = ''
            for i in range(min(4, len(path_parts))):
                curr_path = curr_path + path_parts[i] + '/'
                yield curr_path

        protocol, address_str = urllib.splittype(url)
        host, path = urllib.splithost(address_str)
        user, host = urllib.splituser(host)
        host, port = urllib.splitport(host)
        host = host.strip('/')
        seen_permutations = set()
        for h in url_host_permutations(host):
            for p in url_path_permutations(path):
                u = '{}{}'.format(h, p)
                if u not in seen_permutations:
                    yield u
                    seen_permutations.add(u)


Instruction:
Implement a function in Python 3 to
compare two version strings and return - 1 0 or 1 depending on the equality of the version numbers.

Response:
def _compare_versions(v1, v2):
    """
    Compare two version strings and return -1, 0 or 1 depending on the equality
    of the subset of matching version numbers.

    The implementation is inspired by the top answer at
    http://stackoverflow.com/a/1714190/997768.
    """
    def normalize(v):
        # strip trailing .0 or .00 or .0.0 or ...
        v = re.sub(r'(\.0+)*$', '', v)
        result = []
        for part in v.split('.'):
            # just digits
            m = re.match(r'^(\d+)$', part)
            if m:
                result.append(int(m.group(1)))
                continue
            # digits letters
            m = re.match(r'^(\d+)([a-zA-Z]+)$', part)
            if m:
                result.append(int(m.group(1)))
                result.append(m.group(2))
                continue
            # digits letters digits
            m = re.match(r'^(\d+)([a-zA-Z]+)(\d+)$', part)
            if m:
                result.append(int(m.group(1)))
                result.append(m.group(2))
                result.append(int(m.group(3)))
                continue
        return tuple(result)

    n1 = normalize(v1)
    n2 = normalize(v2)

    return (n1 > n2) - (n1 < n2)


Instruction:
Create a Python 3 function for
splitting version specifier into a tuple of 2 and 1.

Response:
def _split_version_specifier(spec):
    """Splits version specifiers in the form ">= 0.1.2" into ('0.1.2', '>=')"""
    m = re.search(r'([<>=]?=?)?\s*([0-9.a-zA-Z]+)', spec)
    return m.group(2), m.group(1)


Instruction:
Explain what the following Python 3 code does
def exists(package):
    pkg_config_exe = os.environ.get('PKG_CONFIG', None) or 'pkg-config'
    cmd = '{0} --exists {1}'.format(pkg_config_exe, package).split()
    return call(cmd) == 0

Response: Check if package information is available.


Instruction:
Explain what the following Python 3 code does
def libs(package, static=False):
    _raise_if_not_exists(package)
    return _query(package, *_build_options('--libs', static=static))

Response: Return the LDFLAGS string returned by pkg - config.


Instruction:
How would you implement a function in Python 3 that
returns a dictionary of all the variables defined in the. pc pkg - config file of package.

Response:
def variables(package):
    """
    Return a dictionary of all the variables defined in the .pc pkg-config file
    of 'package'.
    """
    _raise_if_not_exists(package)
    result = _query(package, '--print-variables')
    names = (x.strip() for x in result.split('\n') if x != '')
    return dict(((x, _query(package, '--variable={0}'.format(x)).strip()) for x in names))


Instruction:
Can you create a Python 3 function that
checks if the package meets the required version.

Response:
def installed(package, version):
    """
    Check if the package meets the required version.

    The version specifier consists of an optional comparator (one of =, ==, >,
    <, >=, <=) and an arbitrarily long version number separated by dots. The
    should be as you would expect, e.g. for an installed version '0.1.2' of
    package 'foo':

    >>> installed('foo', '==0.1.2')
    True
    >>> installed('foo', '<0.1')
    False
    >>> installed('foo', '>= 0.0.4')
    True

    If ``pkg-config`` not on path, raises ``EnvironmentError``.
    """
    if not exists(package):
        return False

    number, comparator = _split_version_specifier(version)
    modversion = _query(package, '--modversion')

    try:
        result = _compare_versions(modversion, number)
    except ValueError:
        msg = "{0} is not a correct version specifier".format(version)
        raise ValueError(msg)

    if comparator in ('', '=', '=='):
        return result == 0

    if comparator == '>':
        return result > 0

    if comparator == '>=':
        return result >= 0

    if comparator == '<':
        return result < 0

    if comparator == '<=':
        return result <= 0


Instruction:
Can you generate a brief explanation for the following Python 3 code
def parse(packages, static=False):
    for package in packages.split():
        _raise_if_not_exists(package)

    out = _query(packages, *_build_options('--cflags --libs', static=static))
    out = out.replace('\\"', '')
    result = collections.defaultdict(list)

    for token in re.split(r'(?<!\\) ', out):
        key = _PARSE_MAP.get(token[:2])
        if key:
            result[key].append(token[2:].strip())

    def split(m):
        t = tuple(m.split('='))
        return t if len(t) > 1 else (t[0], None)

    result['define_macros'] = [split(m) for m in result['define_macros']]

    # only have members with values not being the empty list (which is default
    # anyway):
    return collections.defaultdict(list, ((k, v) for k, v in result.items() if v))

Response: Parse the output from pkg - config about the passed package or packages.


Instruction:
Write a Python 3 function that can
send the data over UDP while taking the sample_rate in account

Response:
def send(self, data, sample_rate=None):
        '''Send the data over UDP while taking the sample_rate in account

        The sample rate should be a number between `0` and `1` which indicates
        the probability that a message will be sent. The sample_rate is also
        communicated to `statsd` so it knows what multiplier to use.

        :keyword data: The data to send
        :type data: dict
        :keyword sample_rate: The sample rate, defaults to `1` (meaning always)
        :type sample_rate: int
        '''
        if self._disabled:
            self.logger.debug('Connection disabled, not sending data')
            return False
        if sample_rate is None:
            sample_rate = self._sample_rate

        sampled_data = {}
        if sample_rate < 1:
            if random.random() <= sample_rate:
                # Modify the data so statsd knows our sample_rate
                for stat, value in compat.iter_dict(data):
                    sampled_data[stat] = '%s|@%s' % (data[stat], sample_rate)
        else:
            sampled_data = data

        try:
            for stat, value in compat.iter_dict(sampled_data):
                send_data = ('%s:%s' % (stat, value)).encode("utf-8")
                self.udp_sock.send(send_data)
            return True
        except Exception as e:
            self.logger.exception('unexpected error %r while sending data', e)
            return False


Instruction:
Implement a Python 3 function for
starting the timer and store the start time

Response:
def start(self):
        '''Start the timer and store the start time, this can only be executed
        once per instance

        It returns the timer instance so it can be chained when instantiating
        the timer instance like this:
        ``timer = Timer('application_name').start()``'''
        assert self._start is None, (
            'Unable to start, the timer is already running')
        self._last = self._start = time.time()
        return self


Instruction:
How would you explain what the following Python 3 function does
def send(self, subname, delta):
        '''Send the data to statsd via self.connection

        :keyword subname: The subname to report the data to (appended to the
            client name)
        :type subname: str
        :keyword delta: The time delta (time.time() - time.time()) to report
        :type delta: float
        '''
        ms = delta * 1000
        if ms > self.min_send_threshold:
            name = self._get_name(self.name, subname)
            self.logger.info('%s: %0.08fms', name, ms)
            return statsd.Client._send(self, {name: '%0.08f|ms' % ms})
        else:
            return True

Response: Send the data to statsd via self. connection


Instruction:
How would you explain what the following Python 3 function does
def intermediate(self, subname):
        '''Send the time that has passed since our last measurement

        :keyword subname: The subname to report the data to (appended to the
            client name)
        :type subname: str
        '''
        t = time.time()
        response = self.send(subname, t - self._last)
        self._last = t
        return response

Response: Send the time that has passed since our last measurement


Instruction:
How would you implement a function in Python 3 that
stops the timer and send the total since start was run

Response:
def stop(self, subname='total'):
        '''Stop the timer and send the total since `start()` was run

        :keyword subname: The subname to report the data to (appended to the
            client name)
        :type subname: str
        '''
        assert self._stop is None, (
            'Unable to stop, the timer is already stopped')
        self._stop = time.time()
        return self.send(subname, self._stop - self._start)


Instruction:
Can you tell what is the following Python 3 function doing
def decorate(self, function_or_name):
        '''Decorate a function to time the execution

        The method can be called with or without a name. If no name is given
        the function defaults to the name of the function.

        :keyword function_or_name: The name to post to or the function to wrap

        >>> from statsd import Timer
        >>> timer = Timer('application_name')
        >>>
        >>> @timer.decorate
        ... def some_function():
        ...     # resulting timer name: application_name.some_function
        ...     pass
        >>>
        >>> @timer.decorate('my_timer')
        ... def some_other_function():
        ...     # resulting timer name: application_name.my_timer
        ...     pass

        '''
        if callable(function_or_name):
            return self._decorate(function_or_name.__name__, function_or_name)
        else:
            return partial(self._decorate, function_or_name)

Response: Decorate a function to time the execution


Instruction:
Create a Python 3 function for
returning a context manager to time execution of a block of code.

Response:
def time(self, subname=None, class_=None):
        '''Returns a context manager to time execution of a block of code.

        :keyword subname: The subname to report data to
        :type subname: str
        :keyword class_: The :class:`~statsd.client.Client` subclass to use
            (e.g. :class:`~statsd.timer.Timer` or
            :class:`~statsd.counter.Counter`)
        :type class_: :class:`~statsd.client.Client`

        >>> from statsd import Timer
        >>> timer = Timer('application_name')
        >>>
        >>> with timer.time():
        ...     # resulting timer name: application_name
        ...     pass
        >>>
        >>>
        >>> with timer.time('context_timer'):
        ...     # resulting timer name: application_name.context_timer
        ...     pass

        '''
        if class_ is None:
            class_ = Timer
        timer = self.get_client(subname, class_)
        timer.start()
        yield
        timer.stop('')


Instruction:
Create a Python 3 function for
sending the data to statsd via self. connection

Response:
def send(self, subname, value, timestamp=None):
        '''Send the data to statsd via self.connection

        :keyword subname: The subname to report the data to (appended to the
            client name)
        :type subname: str
        :keyword value: The raw value to send
        '''
        if timestamp is None:
            ts = int(dt.datetime.now().strftime("%s"))
        else:
            ts = timestamp
        name = self._get_name(self.name, subname)
        self.logger.info('%s: %s %s' % (name, value, ts))
        return statsd.Client._send(self, {name: '%s|r|%s' % (value, ts)})


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _send(self, subname, value):
        '''Send the data to statsd via self.connection

        :keyword subname: The subname to report the data to (appended to the
            client name)
        :type subname: str
        :keyword value: The gauge value to send
        '''
        name = self._get_name(self.name, subname)
        self.logger.info('%s: %s', name, value)
        return statsd.Client._send(self, {name: '%s|g' % value})

Response: Send the data to statsd via self. connection


Instruction:
Write a Python 3 function for
sending the data to statsd via self. connection

Response:
def send(self, subname, value):
        '''Send the data to statsd via self.connection

        :keyword subname: The subname to report the data to (appended to the
            client name)
        :type subname: str
        :keyword value: The gauge value to send
        '''
        assert isinstance(value, compat.NUM_TYPES)
        return self._send(subname, value)


Instruction:
Explain what the following Python 3 code does
def increment(self, subname=None, delta=1):
        '''Increment the gauge with `delta`

        :keyword subname: The subname to report the data to (appended to the
            client name)
        :type subname: str
        :keyword delta: The delta to add to the gauge
        :type delta: int

        >>> gauge = Gauge('application_name')
        >>> gauge.increment('gauge_name', 10)
        True
        >>> gauge.increment(delta=10)
        True
        >>> gauge.increment('gauge_name')
        True
        '''
        delta = int(delta)
        sign = "+" if delta >= 0 else ""
        return self._send(subname, "%s%d" % (sign, delta))

Response: Increment the gauge with delta


Instruction:
Can you tell what is the following Python 3 function doing
def decrement(self, subname=None, delta=1):
        '''Decrement the gauge with `delta`

        :keyword subname: The subname to report the data to (appended to the
            client name)
        :type subname: str
        :keyword delta: The delta to remove from the gauge
        :type delta: int

        >>> gauge = Gauge('application_name')
        >>> gauge.decrement('gauge_name', 10)
        True
        >>> gauge.decrement(delta=10)
        True
        >>> gauge.decrement('gauge_name')
        True
        '''
        delta = -int(delta)
        sign = "+" if delta >= 0 else ""
        return self._send(subname, "%s%d" % (sign, delta))

Response: Decrement the gauge with delta


Instruction:
How would you explain what the following Python 3 function does
def set(self, subname, value):
        '''
        Set the data ignoring the sign, ie set("test", -1) will set "test"
        exactly to -1 (not decrement it by 1)

        See https://github.com/etsy/statsd/blob/master/docs/metric_types.md
        "Adding a sign to the gauge value will change the value, rather
        than setting it.

            gaugor:-10|g
            gaugor:+4|g

        So if gaugor was 333, those commands would set it to 333 - 10 + 4, or
        327.

        Note: This implies you can't explicitly set a gauge to a negative
        number without first setting it to zero."

        :keyword subname: The subname to report the data to (appended to the
            client name)
        :type subname: str
        :keyword value: The new gauge value
        '''

        assert isinstance(value, compat.NUM_TYPES)
        if value < 0:
            self._send(subname, 0)
        return self._send(subname, value)

Response: Set the value of the gauge entry in the specified subname.


Instruction:
Can you implement a function in Python 3 that
gets a sub - client with a separate namespace

Response:
def get_client(self, name=None, class_=None):
        '''Get a (sub-)client with a separate namespace
        This way you can create a global/app based client with subclients
        per class/function

        :keyword name: The name to use, if the name for this client was `spam`
            and the `name` argument is `eggs` than the resulting name will be
            `spam.eggs`
        :type name: str
        :keyword class_: The :class:`~statsd.client.Client` subclass to use
            (e.g. :class:`~statsd.timer.Timer` or
            :class:`~statsd.counter.Counter`)
        :type class_: :class:`~statsd.client.Client`
        '''

        # If the name was given, use it. Otherwise simply clone
        name = self._get_name(self.name, name)

        # Create using the given class, or the current class
        if not class_:
            class_ = self.__class__

        return class_(
            name=name,
            connection=self.connection,
        )


Instruction:
Here you have a function in Python 3, explain what it does
def get_average(self, name=None):
        '''Shortcut for getting an :class:`~statsd.average.Average` instance

        :keyword name: See :func:`~statsd.client.Client.get_client`
        :type name: str
        '''
        return self.get_client(name=name, class_=statsd.Average)

Response: Shortcut for getting an average. Average instance


Instruction:
Can you generate a brief explanation for the following Python 3 code
def init_git_version(cls, v_str):
        if v_str is None:
            cls._git_version = None
            return

        v_str = v_str.strip()
        try:
            version = cls._git_version = tuple(
                int(x) for x in v_str.split()[2].split('.')[:3])
        except Exception:
            raise ValueError("Could not parse git version output %r. Please "
                             "report this" % v_str)
        return version

Response: Parse and store the parsed version tuple on self.


Instruction:
Write a Python 3 script for
querying remote repo about given ref.

Response:
def query_remote_ref(self, remote, ref):
        """Query remote repo about given ref.
        :return: ``('tag', sha)`` if ref is a tag in remote
                 ``('branch', sha)`` if ref is branch (aka "head") in remote
                 ``(None, ref)`` if ref does not exist in remote. This happens
                 notably if ref if a commit sha (they can't be queried)
        """
        out = self.log_call(['git', 'ls-remote', remote, ref],
                            cwd=self.cwd,
                            callwith=subprocess.check_output).strip()
        for sha, fullref in (l.split() for l in out.splitlines()):
            if fullref == 'refs/heads/' + ref:
                return 'branch', sha
            elif fullref == 'refs/tags/' + ref:
                return 'tag', sha
            elif fullref == ref and ref == 'HEAD':
                return 'HEAD', sha
        return None, ref


Instruction:
Can you write a function in Python 3 where it
wraps a subprocess call with logging.

Response:
def log_call(self, cmd, callwith=subprocess.check_call,
                 log_level=logging.DEBUG, **kw):
        """Wrap a subprocess call with logging
        :param meth: the calling method to use.
        """
        logger.log(log_level, "%s> call %r", self.cwd, cmd)
        ret = callwith(cmd, **kw)
        if callwith == subprocess.check_output:
            ret = console_to_str(ret)
        return ret


Instruction:
Here you have a function in Python 3, explain what it does
def aggregate(self):
        logger.info('Start aggregation of %s', self.cwd)
        target_dir = self.cwd

        is_new = not os.path.exists(target_dir)
        if is_new:
            self.init_repository(target_dir)

        self._switch_to_branch(self.target['branch'])
        for r in self.remotes:
            self._set_remote(**r)
        self.fetch()
        merges = self.merges
        if not is_new:
            # reset to the first merge
            origin = merges[0]
            merges = merges[1:]
            self._reset_to(origin["remote"], origin["ref"])
        for merge in merges:
            self._merge(merge)
        self._execute_shell_command_after()
        logger.info('End aggregation of %s', self.cwd)

Response: Aggregate all merges into the target branch


Instruction:
Can you implement a function in Python 3 that
checks repo status and raise exception if dirty.

Response:
def _check_status(self):
        """Check repo status and except if dirty."""
        logger.info('Checking repo status')
        status = self.log_call(
            ['git', 'status', '--porcelain'],
            callwith=subprocess.check_output,
            cwd=self.cwd,
        )
        if status:
            raise DirtyException(status)


Instruction:
Explain what the following Python 3 code does
def _fetch_options(self, merge):
        cmd = tuple()
        for option in FETCH_DEFAULTS:
            value = merge.get(option, self.defaults.get(option))
            if value:
                cmd += ("--%s" % option, str(value))
        return cmd

Response: Get the fetch options from the given merge dict.


Instruction:
Make a summary of the following Python 3 code
def _set_remote(self, name, url):
        remotes = self._get_remotes()
        exising_url = remotes.get(name)
        if exising_url == url:
            logger.info('Remote already exists %s <%s>', name, url)
            return
        if not exising_url:
            logger.info('Adding remote %s <%s>', name, url)
            self.log_call(['git', 'remote', 'add', name, url], cwd=self.cwd)
        else:
            logger.info('Remote remote %s <%s> -> <%s>',
                        name, exising_url, url)
            self.log_call(['git', 'remote', 'rm', name], cwd=self.cwd)
            self.log_call(['git', 'remote', 'add', name, url], cwd=self.cwd)

Response:
Add remote to the repository. It s equivalent to the command
        git remote add <name > url


Instruction:
Implement a Python 3 function for
collecting all pending merge PRs info.

Response:
def collect_prs_info(self):
        """Collect all pending merge PRs info.

        :returns: mapping of PRs by state
        """
        REPO_RE = re.compile(
            '^(https://github.com/|git@github.com:)'
            '(?P<owner>.*?)/(?P<repo>.*?)(.git)?$')
        PULL_RE = re.compile(
            '^(refs/)?pull/(?P<pr>[0-9]+)/head$')
        remotes = {r['name']: r['url'] for r in self.remotes}
        all_prs = {}
        for merge in self.merges:
            remote = merge['remote']
            ref = merge['ref']
            repo_url = remotes[remote]
            repo_mo = REPO_RE.match(repo_url)
            if not repo_mo:
                logger.debug('%s is not a github repo', repo_url)
                continue
            pull_mo = PULL_RE.match(ref)
            if not pull_mo:
                logger.debug('%s is not a github pull reqeust', ref)
                continue
            pr_info = {
                'owner': repo_mo.group('owner'),
                'repo': repo_mo.group('repo'),
                'pr': pull_mo.group('pr'),
            }
            pr_info['path'] = '{owner}/{repo}/pulls/{pr}'.format(**pr_info)
            pr_info['url'] = 'https://github.com/{path}'.format(**pr_info)
            pr_info['shortcut'] = '{owner}/{repo}#{pr}'.format(**pr_info)
            r = self._github_api_get('/repos/{path}'.format(**pr_info))
            if r.status_code != 200:
                logger.warning(
                    'Could not get status of {path}. '
                    'Reason: {r.status_code} {r.reason}'.format(r=r, **pr_info)
                )
                continue
            pr_info['state'] = r.json().get('state')
            pr_info['merged'] = (
                not r.json().get('merged') and 'not ' or ''
            ) + 'merged'
            all_prs.setdefault(pr_info['state'], []).append(pr_info)
        return all_prs


Instruction:
Can you tell what is the following Python 3 function doing
def show_closed_prs(self):
        all_prs = self.collect_prs_info()
        for pr_info in all_prs.get('closed', []):
            logger.info(
                '{url} in state {state} ({merged})'.format(**pr_info)
            )

Response: Log only closed PRs.


Instruction:
Create a Python 3 function for
logging all PRs grouped by state.

Response:
def show_all_prs(self):
        """Log all PRs grouped by state."""
        for __, prs in self.collect_prs_info().items():
            for pr_info in prs:
                logger.info(
                    '{url} in state {state} ({merged})'.format(**pr_info)
                )


Instruction:
Given the following Python 3 function, write the documentation
def get_repos(config, force=False):
    repo_list = []
    for directory, repo_data in config.items():
        if not os.path.isabs(directory):
            directory = os.path.abspath(directory)
        repo_dict = {
            'cwd': directory,
            'defaults': repo_data.get('defaults', dict()),
            'force': force,
        }
        remote_names = set()
        if 'remotes' in repo_data:
            repo_dict['remotes'] = []
            remotes_data = repo_data['remotes'] or {}
            for remote_name, url in remotes_data.items():
                if not url:
                    raise ConfigException(
                        '%s: No url defined for remote %s.' %
                        (directory, remote_name))
                remote_dict = {
                    'name': remote_name,
                    'url': url
                }
                repo_dict['remotes'].append(remote_dict)
                remote_names.add(remote_name)
            if not remote_names:
                raise ConfigException(
                    '%s: You should at least define one remote.' % directory)
        else:
            raise ConfigException('%s: remotes is not defined.' % directory)
        if 'merges' in repo_data:
            merges = []
            merge_data = repo_data.get('merges') or []
            for merge in merge_data:
                try:
                    # Assume parts is a str
                    parts = merge.split(' ')
                    if len(parts) != 2:
                        raise ConfigException(
                            '%s: Merge must be formatted as '
                            '"remote_name ref".' % directory)
                    merge = {
                        "remote": parts[0],
                        "ref": parts[1],
                    }
                except AttributeError:
                    # Parts is a dict
                    try:
                        merge["remote"] = str(merge["remote"])
                        merge["ref"] = str(merge["ref"])
                    except KeyError:
                        raise ConfigException(
                            '%s: Merge lacks mandatory '
                            '`remote` or `ref` keys.' % directory)
                # Check remote is available
                if merge["remote"] not in remote_names:
                    raise ConfigException(
                        '%s: Merge remote %s not defined in remotes.' %
                        (directory, merge["remote"]))
                merges.append(merge)
            repo_dict['merges'] = merges
            if not merges:
                raise ConfigException(
                    '%s: You should at least define one merge.' % directory)
        else:
            raise ConfigException(
                '%s: merges is not defined.' % directory)
        # Only fetch required remotes by default
        repo_dict["fetch_all"] = repo_data.get("fetch_all", False)
        if isinstance(repo_dict["fetch_all"], string_types):
            repo_dict["fetch_all"] = frozenset((repo_dict["fetch_all"],))
        elif isinstance(repo_dict["fetch_all"], list):
            repo_dict["fetch_all"] = frozenset(repo_dict["fetch_all"])
        if 'target' not in repo_data:
            raise ConfigException('%s: No target defined.' % directory)
        parts = (repo_data.get('target') or "") .split(' ')
        if len(parts) != 2:
            raise ConfigException(
                '%s: Target must be formatted as '
                '"remote_name branch_name"' % directory)

        remote_name, branch = repo_data.get('target').split(' ')
        if remote_name not in remote_names:
            raise ConfigException(
                '%s: Target remote %s not defined in remotes.' %
                (directory, remote_name))
        repo_dict['target'] = {
            'remote': remote_name,
            'branch': branch,
        }
        commands = []
        if 'shell_command_after' in repo_data:
            cmds = repo_data['shell_command_after']
            # if str: turn to list
            if cmds:
                if isinstance(cmds, string_types):
                    cmds = [cmds]
                commands = cmds
        repo_dict['shell_command_after'] = commands
        repo_list.append(repo_dict)
    return repo_list

Response: Return a list of repos from a config file.


Instruction:
Can you tell what is the following Python 3 function doing
def load_config(config, expand_env=False, force=False):
    if not os.path.exists(config):
        raise ConfigException('Unable to find configuration file: %s' % config)

    file_extension = os.path.splitext(config)[1][1:]
    conf = kaptan.Kaptan(handler=kaptan.HANDLER_EXT.get(file_extension))

    if expand_env:
        with open(config, 'r') as file_handler:
            config = Template(file_handler.read())
            config = config.substitute(os.environ)

    conf.import_config(config)
    return get_repos(conf.export('dict') or {}, force)

Response: Load a config file and return repos.


Instruction:
Here you have a function in Python 3, explain what it does
def setup_logger(log=None, level=logging.INFO):
    if not log:
        log = logging.getLogger()
    if not log.handlers:
        channel = logging.StreamHandler()
        if level == logging.DEBUG:
            channel.setFormatter(DebugLogFormatter())
        else:
            channel.setFormatter(LogFormatter())

        log.setLevel(level)
        log.addHandler(channel)

Response: Setup logging for CLI use.


Instruction:
Explain what the following Python 3 code does
def get_parser():

    main_parser = argparse.ArgumentParser(
        formatter_class=argparse.RawTextHelpFormatter)

    main_parser.add_argument(
        '-c', '--config',
        dest='config',
        type=str,
        nargs='?',
        help='Pull the latest repositories from config(s)'
    ).completer = argcomplete.completers.FilesCompleter(
        allowednames=('.yaml', '.yml', '.json'), directories=False
    )

    main_parser.add_argument(
        '-p', '--push',
        dest='do_push',
        action='store_true', default=False,
        help='Push result to target',
    )

    main_parser.add_argument(
        '-d', '--dirmatch',
        dest='dirmatch',
        type=str,
        nargs='?',
        help='Pull only from the directories. Accepts fnmatch(1)'
             'by commands'
    )
    main_parser.add_argument(
        '--log-level',
        default='INFO',
        dest='log_level',
        type=_log_level_string_to_int,
        nargs='?',
        help='Set the logging output level. {0}'.format(_LOG_LEVEL_STRINGS))

    main_parser.add_argument(
        '-e', '--expand-env',
        dest='expand_env',
        default=False,
        action='store_true',
        help='Expand environment variables in configuration file',
    )
    main_parser.add_argument(
        '-f', '--force',
        dest='force',
        default=False,
        action='store_true',
        help='Force cleanup and aggregation on dirty repositories.',
    )

    main_parser.add_argument(
        '-j', '--jobs',
        dest='jobs',
        default=1,
        type=int,
        help='Amount of processes to use when aggregating repos. '
             'This is useful when there are a lot of large repos. '
             'Set `1` or less to disable multiprocessing (default).',
    )

    main_parser.add_argument(
        'command',
        nargs='?',
        default='aggregate',
        help='aggregate (default): run the aggregation process.\n'
             'show-all-prs: show GitHub pull requests in merge sections\n'
             '              such pull requests are indentified as having\n'
             '              a github.com remote and a\n'
             '              refs/pull/NNN/head ref in the merge section.\n'
             'show-closed-prs: show pull requests that are not open anymore.\n'
    )

    return main_parser

Response: Return an argument parser for the CLI.