Instruction:
Can you write a function in Python 3 where it
loads and shape data for training and test sets of Keras + Pescador.

Response:
def setup_data():
    """Load and shape data for training with Keras + Pescador.

    Returns
    -------
    input_shape : tuple, len=3
        Shape of each sample; adapts to channel configuration of Keras.

    X_train, y_train : np.ndarrays
        Images and labels for training.

    X_test, y_test : np.ndarrays
        Images and labels for test.
    """
    # The data, shuffled and split between train and test sets
    (x_train, y_train), (x_test, y_test) = mnist.load_data()

    if K.image_data_format() == 'channels_first':
        x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols)
        x_test = x_test.reshape(x_test.shape[0], 1, img_rows, img_cols)
        input_shape = (1, img_rows, img_cols)
    else:
        x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1)
        x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1)
        input_shape = (img_rows, img_cols, 1)

    x_train = x_train.astype('float32')
    x_test = x_test.astype('float32')
    x_train /= 255
    x_test /= 255
    print('x_train shape:', x_train.shape)
    print(x_train.shape[0], 'train samples')
    print(x_test.shape[0], 'test samples')

    # convert class vectors to binary class matrices
    y_train = keras.utils.to_categorical(y_train, num_classes)
    y_test = keras.utils.to_categorical(y_test, num_classes)

    return input_shape, (x_train, y_train), (x_test, y_test)


Instruction:
Explain what the following Python 3 code does
def build_model(input_shape):
    model = Sequential()

    model.add(Conv2D(32, kernel_size=(3, 3),
                     activation='relu',
                     input_shape=input_shape))
    model.add(Conv2D(64, kernel_size=(3, 3),
                     activation='relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(0.25))

    model.add(Flatten())
    model.add(Dense(128, activation='relu'))
    model.add(Dropout(0.5))
    model.add(Dense(num_classes, activation='softmax'))

    model.compile(loss=keras.losses.categorical_crossentropy,
                  optimizer=keras.optimizers.Adadelta(),
                  metrics=['accuracy'])
    return model

Response: Create a compiled Keras model.


Instruction:
Can you generate the documentation for the following Python 3 function
def additive_noise(stream, key='X', scale=1e-1):
    '''Add noise to a data stream.

    Parameters
    ----------
    stream : iterable
        A stream that yields data objects.

    key : string, default='X'
        Name of the field to add noise.

    scale : float, default=0.1
        Scale factor for gaussian noise.

    Yields
    ------
    data : dict
        Updated data objects in the stream.
    '''
    for data in stream:
        noise_shape = data[key].shape
        noise = scale * np.random.randn(*noise_shape)
        data[key] = data[key] + noise
        yield data

Response: Add noise to a data stream.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def parse_denovo_params(user_params=None):
    config = MotifConfig()

    if user_params is None:
        user_params = {}
    params = config.get_default_params()
    params.update(user_params)

    if params.get("torque"):
        logger.debug("Using torque")
    else:
        logger.debug("Using multiprocessing")

    params["background"] = [x.strip() for x in params["background"].split(",")]

    logger.debug("Parameters:")
    for param, value in params.items():
        logger.debug("  %s: %s", param, value)

    # Maximum time?

    if params["max_time"]:
        try:
            max_time = params["max_time"] = float(params["max_time"])
        except Exception:
            logger.debug("Could not parse max_time value, setting to no limit")
            params["max_time"] = -1

    if params["max_time"] > 0:
        logger.debug("Time limit for motif prediction: %0.2f hours", max_time)
        params["max_time"] = 3600 * params["max_time"]
        logger.debug("Max_time in seconds %0.0f", max_time)
    else:
        logger.debug("No time limit for motif prediction")

    return params

Response:
Return default GimmeMotifs parameters.

    Defaults will be replaced with parameters defined in user_params.

    Parameters
    ----------
    user_params : dict, optional
        User-defined parameters.

    Returns
    -------
    params : dict


Instruction:
Can you generate a brief explanation for the following Python 3 code
def rankagg_R(df, method="stuart"):
    tmpdf = NamedTemporaryFile()
    tmpscript = NamedTemporaryFile(mode="w")
    tmpranks = NamedTemporaryFile()

    df.to_csv(tmpdf.name, sep="\t",index=False)

    script = '''
library(RobustRankAggreg);
a = read.table("{}", header=TRUE);
x = lapply(a, as.vector);
result = aggregateRanks(x, method="{}");
result$p.adjust = p.adjust(result$Score);
 write.table(result, file="{}", sep="\t", quote=FALSE, row.names=FALSE);
'''.format(tmpdf.name, method, tmpranks.name)
    tmpscript.write(script)
    tmpscript.flush()

    p = sp.Popen(["Rscript", tmpscript.name], stdout=sp.PIPE, stderr=sp.PIPE)
    stderr, stdout = p.communicate()
    df = pd.read_table(tmpranks.name, index_col=0)
    return df["p.adjust"]

Response: Return aggregated ranks as implemented in the RobustRankAgg R package.


Instruction:
Can you write a function in Python 3 where it
returns aggregated ranks. Implementation is ported from the RobustRankAggreg R package References: Kolde et al., 2012, DOI: 10.1093/bioinformatics/btr709 Stuart et al., 2003, DOI: 10.1126/science.1087447 Parameters ---------- df : pandas.DataFrame DataFrame with values to be ranked and aggregated Returns ------- pandas.DataFrame with aggregated ranks

Response:
def rankagg(df, method="stuart"):
    """Return aggregated ranks.

    Implementation is ported from the RobustRankAggreg R package

    References:
        Kolde et al., 2012, DOI: 10.1093/bioinformatics/btr709
        Stuart et al., 2003,  DOI: 10.1126/science.1087447

    Parameters
    ----------
    df : pandas.DataFrame
        DataFrame with values to be ranked and aggregated

    Returns
    -------
    pandas.DataFrame with aggregated ranks
    """
    rmat = pd.DataFrame(index=df.iloc[:,0])

    step = 1 / rmat.shape[0]
    for col in df.columns:
        rmat[col] = pd.DataFrame({col:np.arange(step, 1 + step, step)}, index=df[col]).loc[rmat.index]
    rmat = rmat.apply(sorted, 1, result_type="expand")
    p = rmat.apply(qStuart, 1)
    df = pd.DataFrame(
        {"p.adjust":multipletests(p, method="h")[1]},
        index=rmat.index).sort_values('p.adjust')

    return df["p.adjust"]


Instruction:
Can you create a Python 3 function that
yields data while optionally burning compute cycles.

Response:
def data_gen(n_ops=100):
    """Yield data, while optionally burning compute cycles.

    Parameters
    ----------
    n_ops : int, default=100
        Number of operations to run between yielding data.

    Returns
    -------
    data : dict
        A object which looks like it might come from some
        machine learning problem, with X as features, and y as targets.
    """
    while True:
        X = np.random.uniform(size=(64, 64))
        yield dict(X=costly_function(X, n_ops),
                   y=np.random.randint(10, size=(1,)))


Instruction:
Given the following Python 3 function, write the documentation
def mp_calc_stats(motifs, fg_fa, bg_fa, bg_name=None):
    try:
        stats = calc_stats(motifs, fg_fa, bg_fa, ncpus=1)
    except Exception as e:
        raise
        sys.stderr.write("ERROR: {}\n".format(str(e)))
        stats = {}

    if not bg_name:
        bg_name = "default"

    return bg_name, stats

Response: Parallel calculation of motif statistics.


Instruction:
Can you tell what is the following Python 3 function doing
def pp_predict_motifs(fastafile, outfile, analysis="small", organism="hg18", single=False, background="", tools=None, job_server=None, ncpus=8, max_time=-1, stats_fg=None, stats_bg=None):
    if tools is None:
        tools = {}

    config = MotifConfig()

    if not tools:
        tools = dict([(x,1) for x in config.get_default_params["tools"].split(",")])

    #logger = logging.getLogger('gimme.prediction.pp_predict_motifs')

    wmin = 5
    step = 1
    if analysis in ["large","xl"]:
        step = 2
        wmin = 6

    analysis_max = {"xs":5,"small":8, "medium":10,"large":14, "xl":20}
    wmax = analysis_max[analysis]

    if analysis == "xs":
        sys.stderr.write("Setting analysis xs to small")
        analysis = "small"

    if not job_server:
        n_cpus = int(config.get_default_params()["ncpus"])
        job_server = Pool(processes=n_cpus, maxtasksperchild=1000)

    jobs = {}

    result = PredictionResult(
                outfile,
                fg_file=stats_fg,
                background=stats_bg,
                job_server=job_server,
                )

    # Dynamically load all tools
    toolio = [x[1]() for x in inspect.getmembers(
                                                tool_classes,
                                                lambda x:
                                                        inspect.isclass(x) and
                                                        issubclass(x, tool_classes.MotifProgram)
                                                ) if x[0] != 'MotifProgram']

    # TODO:
    # Add warnings for running time: Weeder, GADEM

    ### Add all jobs to the job_server ###
    params = {
            'analysis': analysis,
            'background':background,
            "single":single,
            "organism":organism
            }

    # Tools that don't use a specified width usually take longer
    # ie. GADEM, XXmotif, MEME
    # Start these first.
    for t in [tool for tool in toolio if not tool.use_width]:
        if t.name in tools and tools[t.name]:
            logger.debug("Starting %s job", t.name)
            job_name = t.name
            jobs[job_name] = job_server.apply_async(
                        _run_tool,
                        (job_name, t, fastafile, params),
                        callback=result.add_motifs)
        else:
            logger.debug("Skipping %s", t.name)

    for t in [tool for tool in toolio if tool.use_width]:
        if t.name in tools and tools[t.name]:
            for i in range(wmin, wmax + 1, step):
                logger.debug("Starting %s job, width %s", t.name, i)
                job_name = "%s_width_%s" % (t.name, i)
                my_params = params.copy()
                my_params['width'] = i
                jobs[job_name] = job_server.apply_async(
                    _run_tool,
                    (job_name, t, fastafile, my_params),
                    callback=result.add_motifs)
        else:
            logger.debug("Skipping %s", t.name)

    logger.info("all jobs submitted")
    for job in jobs.values():
        job.get()

    result.wait_for_stats()
    ### Wait until all jobs are finished or the time runs out ###
#    start_time = time()
#    try:
#        # Run until all jobs are finished
#        while len(result.finished) < len(jobs.keys()) and (not(max_time) or time() - start_time < max_time):
#            pass
#        if len(result.finished) < len(jobs.keys()):
#            logger.info("Maximum allowed running time reached, destroying remaining jobs")
#            job_server.terminate()
#            result.submit_remaining_stats()
#    ### Or the user gets impatient... ###
#    except KeyboardInterrupt:
#        # Destroy all running jobs
#        logger.info("Caught interrupt, destroying all running jobs")
#        job_server.terminate()
#        result.submit_remaining_stats()
#
#
#    if stats_fg and stats_bg:
#        logger.info("waiting for motif statistics")
#        n = 0
#        last_len = 0
#
#
#        while len(set(result.stats.keys())) < len(set([str(m) for m in result.motifs])):
#            if n >= 30:
#                logger.debug("waited long enough")
#                logger.debug("motifs: %s, stats: %s", len(result.motifs), len(result.stats.keys()))
#                for i,motif in enumerate(result.motifs):
#                    if "{}_{}".format(motif.id, motif.to_consensus()) not in result.stats:
#                        logger.debug("deleting %s", motif)
#                        del result.motifs[i]
#                break
#            sleep(2)
#            if len(result.stats.keys()) == last_len:
#                n += 1
#            else:
#                last_len = len(result.stats.keys())
#                n = 0
#
    return result

Response: Predict motifs using multiprocessing.


Instruction:
Here you have a function in Python 3, explain what it does
def predict_motifs(infile, bgfile, outfile, params=None, stats_fg=None, stats_bg=None):

    # Parse parameters
    required_params = ["tools", "available_tools", "analysis",
                                "genome", "use_strand", "max_time"]
    if params is None:
        params = parse_denovo_params()
    else:
        for p in required_params:
            if p not in params:
                params = parse_denovo_params()
                break

    # Define all tools
    tools = dict(
            [
                (x.strip(), x in [y.strip() for y in params["tools"].split(",")])
                    for x in params["available_tools"].split(",")
            ]
            )

    # Predict the motifs
    analysis = params["analysis"]
    logger.info("starting motif prediction (%s)", analysis)
    logger.info("tools: %s",
            ", ".join([x for x in tools.keys() if tools[x]]))
    result = pp_predict_motifs(
                    infile,
                    outfile,
                    analysis,
                    params.get("genome", None),
                    params["use_strand"],
                    bgfile,
                    tools,
                    None,
                    #logger=logger,
                    max_time=params["max_time"],
                    stats_fg=stats_fg,
                    stats_bg=stats_bg
                )

    motifs = result.motifs
    logger.info("predicted %s motifs", len(motifs))
    logger.debug("written to %s", outfile)

    if len(motifs) == 0:
        logger.info("no motifs found")
        result.motifs = []

    return result

Response: Predict motifs for a single file.


Instruction:
Explain what the following Python 3 code does
def add_motifs(self, args):
        self.lock.acquire()
        # Callback function for motif programs
        if args is None or len(args) != 2 or len(args[1]) != 3:
            try:
                job = args[0]
                logger.warn("job %s failed", job)
                self.finished.append(job)
            except Exception:
                logger.warn("job failed")
            return

        job, (motifs, stdout, stderr) = args

        logger.info("%s finished, found %s motifs", job, len(motifs))

        for motif in motifs:
            if self.do_counter:
                self.counter += 1
                motif.id = "gimme_{}_".format(self.counter) + motif.id
            f = open(self.outfile, "a")
            f.write("%s\n" % motif.to_pfm())
            f.close()
            self.motifs.append(motif)

        if self.do_stats and len(motifs) > 0:
            #job_id = "%s_%s" % (motif.id, motif.to_consensus())
            logger.debug("Starting stats job of %s motifs", len(motifs))
            for bg_name, bg_fa in self.background.items():
                job = self.job_server.apply_async(
                                    mp_calc_stats,
                                    (motifs, self.fg_fa, bg_fa, bg_name),
                                    callback=self.add_stats
                                    )
                self.stat_jobs.append(job)

        logger.debug("stdout %s: %s", job, stdout)
        logger.debug("stdout %s: %s", job, stderr)
        self.finished.append(job)
        self.lock.release()

Response: Add motifs to the result object.


Instruction:
Write a Python 3 function that can
wait for all jobs to finish.

Response:
def wait_for_stats(self):
        """Make sure all jobs are finished."""
        logging.debug("waiting for statistics to finish")
        for job in self.stat_jobs:
            job.get()
        sleep(2)


Instruction:
Given the following Python 3 function, write the documentation
def add_stats(self, args):
        bg_name, stats = args
        logger.debug("Stats: %s %s", bg_name, stats)

        for motif_id in stats.keys():
            if motif_id not in self.stats:
                self.stats[motif_id] = {}

            self.stats[motif_id][bg_name] = stats[motif_id]

Response: Callback to add motif statistics.


Instruction:
Can you implement a function in Python 3 that
prepares a narrowPeak file for de novo motif prediction.

Response:
def prepare_denovo_input_narrowpeak(inputfile, params, outdir):
    """Prepare a narrowPeak file for de novo motif prediction.

    All regions to same size; split in test and validation set;
    converted to FASTA.

    Parameters
    ----------
    inputfile : str
        BED file with input regions.

    params : dict
        Dictionary with parameters.

    outdir : str
        Output directory to save files.
    """

    bedfile = os.path.join(outdir, "input.from.narrowpeak.bed")
    p = re.compile(r'^(#|track|browser)')
    width = int(params["width"])
    logger.info("preparing input (narrowPeak to BED, width %s)", width)
    warn_no_summit = True
    with open(bedfile, "w") as f_out:
        with open(inputfile) as f_in:
            for line in f_in:
                if p.search(line):
                    continue
                vals = line.strip().split("\t")
                start, end = int(vals[1]), int(vals[2])
                summit = int(vals[9])
                if summit == -1:
                    if warn_no_summit:
                        logger.warn("No summit present in narrowPeak file, using the peak center.")
                        warn_no_summit = False
                    summit = (end - start) // 2

                start = start + summit - (width // 2)
                end = start + width
                f_out.write("{}\t{}\t{}\t{}\n".format(
                    vals[0],
                    start,
                    end,
                    vals[6]
                    ))

    prepare_denovo_input_bed(bedfile, params, outdir)


Instruction:
Write a Python 3 script to
prepare a BED file for de novo motif prediction.

Response:
def prepare_denovo_input_bed(inputfile, params, outdir):
    """Prepare a BED file for de novo motif prediction.

    All regions to same size; split in test and validation set;
    converted to FASTA.

    Parameters
    ----------
    inputfile : str
        BED file with input regions.

    params : dict
        Dictionary with parameters.

    outdir : str
        Output directory to save files.
    """
    logger.info("preparing input (BED)")

    # Create BED file with regions of equal size
    width = int(params["width"])
    bedfile = os.path.join(outdir, "input.bed")
    write_equalwidth_bedfile(inputfile, width, bedfile)

    abs_max = int(params["abs_max"])
    fraction = float(params["fraction"])
    pred_bedfile = os.path.join(outdir, "prediction.bed")
    val_bedfile = os.path.join(outdir, "validation.bed")
    # Split input into prediction and validation set
    logger.debug(
                "Splitting %s into prediction set (%s) and validation set (%s)",
                bedfile, pred_bedfile, val_bedfile)
    divide_file(bedfile, pred_bedfile, val_bedfile, fraction, abs_max)

    config = MotifConfig()

    genome = Genome(params["genome"])
    for infile in [pred_bedfile, val_bedfile]:
        genome.track2fasta(
            infile,
            infile.replace(".bed", ".fa"),
            )

    # Create file for location plots
    lwidth = int(params["lwidth"])
    extend = (lwidth - width) // 2

    genome.track2fasta(
            val_bedfile,
            os.path.join(outdir, "localization.fa"),
            extend_up=extend,
            extend_down=extend,
            stranded=params["use_strand"],
            )


Instruction:
How would you explain what the following Python 3 function does
def prepare_denovo_input_fa(inputfile, params, outdir):
    fraction = float(params["fraction"])
    abs_max = int(params["abs_max"])

    logger.info("preparing input (FASTA)")

    pred_fa = os.path.join(outdir, "prediction.fa")
    val_fa = os.path.join(outdir, "validation.fa")
    loc_fa = os.path.join(outdir, "localization.fa")

    # Split inputfile in prediction and validation set
    logger.debug(
        "Splitting %s into prediction set (%s) and validation set (%s)",
        inputfile, pred_fa, val_fa)

    divide_fa_file(inputfile, pred_fa, val_fa, fraction, abs_max)

    # File for location plots
    shutil.copy(val_fa, loc_fa)
    seqs = Fasta(loc_fa).seqs
    lwidth = len(seqs[0])
    all_same_width = not(False in [len(seq) == lwidth for seq in seqs])
    if not all_same_width:
        logger.warn(
            "PLEASE NOTE: FASTA file contains sequences of different lengths. "
            "Positional preference plots might be incorrect!")

Response: Prepare the de novo motif prediction and validation files for prediction and validation.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def create_background(bg_type, fafile, outfile, genome="hg18", width=200, nr_times=10, custom_background=None):
    width = int(width)
    config = MotifConfig()
    fg = Fasta(fafile)

    if bg_type in ["genomic", "gc"]:
        if not genome:
            logger.error("Need a genome to create background")
            sys.exit(1)

    if bg_type == "random":
        f = MarkovFasta(fg, k=1, n=nr_times * len(fg))
        logger.debug("Random background: %s", outfile)
    elif bg_type == "genomic":
        logger.debug("Creating genomic background")
        f = RandomGenomicFasta(genome, width, nr_times * len(fg))
    elif bg_type == "gc":
        logger.debug("Creating GC matched background")
        f = MatchedGcFasta(fafile, genome, nr_times * len(fg))
        logger.debug("GC matched background: %s", outfile)
    elif bg_type == "promoter":
        fname = Genome(genome).filename
        gene_file = fname.replace(".fa", ".annotation.bed.gz")
        if not gene_file:
            gene_file = os.path.join(config.get_gene_dir(), "%s.bed" % genome)
        if not os.path.exists(gene_file):
            print("Could not find a gene file for genome {}")
            print("Did you use the --annotation flag for genomepy?")
            print("Alternatively make sure there is a file called {}.bed in {}".format(genome, config.get_gene_dir()))
            raise ValueError()

        logger.info(
                "Creating random promoter background (%s, using genes in %s)",
                genome, gene_file)
        f = PromoterFasta(gene_file, genome, width, nr_times * len(fg))
        logger.debug("Random promoter background: %s", outfile)
    elif bg_type == "custom":
        bg_file = custom_background
        if not bg_file:
            raise IOError(
                    "Background file not specified!")

        if not os.path.exists(bg_file):
            raise IOError(
                    "Custom background file %s does not exist!",
                    bg_file)
        else:
            logger.info("Copying custom background file %s to %s.",
                    bg_file, outfile)
            f = Fasta(bg_file)
            l = np.median([len(seq) for seq in f.seqs])
            if l < (width * 0.95) or l > (width * 1.05):
                   logger.warn(
                    "The custom background file %s contains sequences with a "
                    "median length of %s, while GimmeMotifs predicts motifs in sequences "
                    "of length %s. This will influence the statistics! It is recommended "
                    "to use background sequences of the same length.",
                    bg_file, l, width)

    f.writefasta(outfile)
    return len(f)

Response: Create a new background of a specific type.


Instruction:
Explain what the following Python 3 code does
def create_backgrounds(outdir, background=None, genome="hg38", width=200, custom_background=None):
    if background is None:
        background = ["random"]
        nr_sequences = {}

    # Create background for motif prediction
    if "gc" in background:
        pred_bg = "gc"
    else:
        pred_bg = background[0]

    create_background(
                    pred_bg,
                    os.path.join(outdir, "prediction.fa"),
                    os.path.join(outdir, "prediction.bg.fa"),
                    genome=genome,
                    width=width,
                    custom_background=custom_background)

    # Get background fasta files for statistics
    bg_info = {}
    nr_sequences = {}
    for bg in background:
        fname = os.path.join(outdir, "bg.{}.fa".format(bg))
        nr_sequences[bg] = create_background(
                                        bg,
                                        os.path.join(outdir, "validation.fa"),
                                        fname,
                                        genome=genome,
                                        width=width,
                                        custom_background=custom_background)

        bg_info[bg] = fname
    return bg_info

Response: Create different backgrounds for motif prediction and validation.


Instruction:
Create a Python 3 function for
filtering significant motifs based on several statistics.

Response:
def _is_significant(stats, metrics=None):
    """Filter significant motifs based on several statistics.

    Parameters
    ----------
    stats : dict
        Statistics disctionary object.

    metrics : sequence
        Metric with associated minimum values. The default is
        (("max_enrichment", 3), ("roc_auc", 0.55), ("enr_at_fpr", 0.55))

    Returns
    -------
    significant : bool
    """
    if metrics is None:
        metrics = (("max_enrichment", 3), ("roc_auc", 0.55), ("enr_at_fpr", 0.55))

    for stat_name, min_value in metrics:
        if stats.get(stat_name, 0) < min_value:
            return False

    return True


Instruction:
Make a summary of the following Python 3 code
def filter_significant_motifs(fname, result, bg, metrics=None):
    sig_motifs = []
    with open(fname, "w") as f:
        for motif in result.motifs:
            stats = result.stats.get(
                    "%s_%s" % (motif.id, motif.to_consensus()), {}).get(bg, {}
                    )
            if _is_significant(stats, metrics):
                f.write("%s\n" % motif.to_pfm())
                sig_motifs.append(motif)

    logger.info("%s motifs are significant", len(sig_motifs))
    logger.debug("written to %s", fname)

    return sig_motifs

Response: Filter significant motifs based on several statistics.


Instruction:
How would you code a function in Python 3 to
return the best motif for a single cluster.

Response:
def best_motif_in_cluster(single_pwm, clus_pwm, clusters, fg_fa, background, stats=None, metrics=("roc_auc", "recall_at_fdr")):
    """Return the best motif per cluster for a clustering results.

    The motif can be either the average motif or one of the clustered motifs.

    Parameters
    ----------
    single_pwm : str
        Filename of motifs.

    clus_pwm : str
        Filename of motifs.

    clusters :
        Motif clustering result.

    fg_fa : str
        Filename of FASTA file.

    background : dict
        Dictionary for background file names.

    stats : dict, optional
        If statistics are not supplied they will be computed.

    metrics : sequence, optional
        Metrics to use for motif evaluation. Default are "roc_auc" and
        "recall_at_fdr".

    Returns
    -------
    motifs : list
        List of Motif instances.
    """
    # combine original and clustered motifs
    motifs = read_motifs(single_pwm) + read_motifs(clus_pwm)
    motifs = dict([(str(m), m) for m in motifs])

    # get the statistics for those motifs that were not yet checked
    clustered_motifs = []
    for clus,singles in clusters:
        for motif in set([clus] + singles):
            if str(motif) not in stats:
                clustered_motifs.append(motifs[str(motif)])

    new_stats = {}
    for bg, bg_fa in background.items():
        for m,s in calc_stats(clustered_motifs, fg_fa, bg_fa).items():
            if m not in new_stats:
                new_stats[m] = {}
            new_stats[m][bg] = s
    stats.update(new_stats)

    rank = rank_motifs(stats, metrics)

    # rank the motifs
    best_motifs = []
    for clus, singles in clusters:
        if len(singles) > 1:
            eval_motifs = singles
            if clus not in motifs:
                eval_motifs.append(clus)
            eval_motifs = [motifs[str(e)] for e in eval_motifs]
            best_motif = sorted(eval_motifs, key=lambda x: rank[str(x)])[-1]
            best_motifs.append(best_motif)
        else:
            best_motifs.append(clus)
        for bg in background:
            stats[str(best_motifs[-1])][bg]["num_cluster"] = len(singles)

    best_motifs = sorted(best_motifs, key=lambda x: rank[str(x)], reverse=True)
    return best_motifs


Instruction:
Write a Python 3 script for
renaming motifs to GimmeMotifs_1.. GimmeMotifs_N.

Response:
def rename_motifs(motifs, stats=None):
    """Rename motifs to GimmeMotifs_1..GimmeMotifs_N.

    If stats object is passed, stats will be copied."""
    final_motifs = []
    for i, motif in enumerate(motifs):
        old = str(motif)
        motif.id = "GimmeMotifs_{}".format(i + 1)
        final_motifs.append(motif)
        if stats:
            stats[str(motif)] = stats[old].copy()

    if stats:
        return final_motifs, stats
    else:
        return final_motifs


Instruction:
Can you write a function in Python 3 where it
registers a class as a decorator for the database.

Response:
def register_db(cls, dbname):
        """Register method to keep list of dbs."""
        def decorator(subclass):
            """Register as decorator function."""
            cls._dbs[dbname] = subclass
            subclass.name = dbname
            return subclass
        return decorator


Instruction:
Implement a Python 3 function for
running a single motif activity prediction algorithm.

Response:
def moap(inputfile, method="hypergeom", scoring=None, outfile=None, motiffile=None, pwmfile=None, genome=None, fpr=0.01, ncpus=None,
        subsample=None):
    """Run a single motif activity prediction algorithm.

    Parameters
    ----------
    inputfile : str
        :1File with regions (chr:start-end) in first column and either cluster
        name in second column or a table with values.

    method : str, optional
        Motif activity method to use. Any of 'hypergeom', 'lasso',
        'lightningclassification', 'lightningregressor', 'bayesianridge',
        'rf', 'xgboost'. Default is 'hypergeom'.

    scoring:  str, optional
        Either 'score' or 'count'

    outfile : str, optional
        Name of outputfile to save the fitted activity values.

    motiffile : str, optional
        Table with motif scan results. First column should be exactly the same
        regions as in the inputfile.

    pwmfile : str, optional
        File with motifs in pwm format. Required when motiffile is not
        supplied.

    genome : str, optional
        Genome name, as indexed by gimme. Required when motiffile is not
        supplied

    fpr : float, optional
        FPR for motif scanning

    ncpus : int, optional
        Number of threads to use. Default is the number specified in the config.

    Returns
    -------
    pandas DataFrame with motif activity
    """

    if scoring and scoring not in ['score', 'count']:
        raise ValueError("valid values are 'score' and 'count'")

    config = MotifConfig()

    if inputfile.endswith("feather"):
        df = pd.read_feather(inputfile)
        df = df.set_index(df.columns[0])
    else:
        # read data
        df = pd.read_table(inputfile, index_col=0, comment="#")

    clf = Moap.create(method, ncpus=ncpus)

    if clf.ptype == "classification":
        if df.shape[1] != 1:
            raise ValueError("1 column expected for {}".format(method))
    else:
        if np.dtype('object') in set(df.dtypes):
            raise ValueError(
                    "columns should all be numeric for {}".format(method))

    if motiffile is None:
        if genome is None:
            raise ValueError("need a genome")

        pwmfile = pwmfile_location(pwmfile)
        try:
            motifs = read_motifs(pwmfile)
        except:
            sys.stderr.write("can't read motifs from {}".format(pwmfile))
            raise

        # initialize scanner
        s = Scanner(ncpus=ncpus)
        sys.stderr.write(pwmfile + "\n")
        s.set_motifs(pwmfile)
        s.set_genome(genome)
        s.set_background(genome=genome)

        # scan for motifs
        sys.stderr.write("scanning for motifs\n")
        motif_names = [m.id for m in read_motifs(pwmfile)]
        scores = []
        if method == 'classic' or scoring == "count":
            s.set_threshold(fpr=fpr)
            for row in s.count(list(df.index)):
                scores.append(row)
        else:
            for row in s.best_score(list(df.index), normalize=True):
                scores.append(row)

        motifs = pd.DataFrame(scores, index=df.index, columns=motif_names)
    else:
        motifs = pd.read_table(motiffile, index_col=0, comment="#")

    if outfile and os.path.exists(outfile):
        out = pd.read_table(outfile, index_col=0, comment="#")
        ncols = df.shape[1]
        if ncols == 1:
            ncols = len(df.iloc[:,0].unique())

        if out.shape[0] == motifs.shape[1] and out.shape[1] == ncols:
            logger.warn("%s output already exists... skipping", method)
            return out

    if subsample is not None:
        n = int(subsample * df.shape[0])
        logger.debug("Subsampling %d regions", n)
        df = df.sample(n)

    motifs = motifs.loc[df.index]

    if method == "lightningregressor":
        outdir = os.path.dirname(outfile)
        tmpname = os.path.join(outdir, ".lightning.tmp")
        clf.fit(motifs, df, tmpdir=tmpname)
        shutil.rmtree(tmpname)
    else:
        clf.fit(motifs, df)

    if outfile:
        with open(outfile, "w") as f:
            f.write("# maelstrom - GimmeMotifs version {}\n".format(__version__))
            f.write("# method: {} with motif {}\n".format(method, scoring))
            if genome:
                f.write("# genome: {}\n".format(genome))
            if motiffile:
                f.write("# motif table: {}\n".format(motiffile))
            f.write("# {}\n".format(clf.act_description))

        with open(outfile, "a") as f:
            clf.act_.to_csv(f, sep="\t")

    return clf.act_


Instruction:
Create a Python 3 function for
creating a Moap instance based on the predictor name.

Response:
def create(cls, name, ncpus=None):
        """Create a Moap instance based on the predictor name.

        Parameters
        ----------
        name : str
            Name of the predictor (eg. Xgboost, BayesianRidge, ...)

        ncpus : int, optional
            Number of threads. Default is the number specified in the config.

        Returns
        -------
        moap : Moap instance
            moap instance.
        """
        try:
            return cls._predictors[name.lower()](ncpus=ncpus)
        except KeyError:
            raise Exception("Unknown class")


Instruction:
Make a summary of the following Python 3 code
def register_predictor(cls, name):
        def decorator(subclass):
            """Register as decorator function."""
            cls._predictors[name.lower()] = subclass
            subclass.name = name.lower()
            return subclass
        return decorator

Response: Register a predictor with the given name.


Instruction:
How would you implement a function in Python 3 that
lists available classification predictors.

Response:
def list_classification_predictors(self):
        """List available classification predictors."""
        preds = [self.create(x) for x in self._predictors.keys()]
        return [x.name for x in preds if x.ptype == "classification"]


Instruction:
Given the following Python 3 function, write the documentation
def iterate(self, max_iter=None):
        '''Instantiate an iterator.

        Parameters
        ----------
        max_iter : None or int > 0
            Maximum number of iterations to yield.
            If ``None``, exhaust the stream.

        Yields
        ------
        obj : Objects yielded by the streamer provided on init.

        See Also
        --------
        cycle : force an infinite stream.

        '''
        # Use self as context manager / calls __enter__() => _activate()
        with self as active_streamer:
            for n, obj in enumerate(active_streamer.stream_):
                if max_iter is not None and n >= max_iter:
                    break
                yield obj

Response:
Instantiate an iterator.

        Parameters
        ----------
        max_iter : None or int > 0
            Maximum number of iterations to yield.
            If ``None``, exhaust the stream.

        Yields
        ------
        obj : Objects yielded by the streamer provided on init.

        See Also
        --------
        cycle : force an infinite stream.


Instruction:
Can you generate the documentation for the following Python 3 function
def cycle(self, max_iter=None):
        '''Iterate from the streamer infinitely.

        This function will force an infinite stream, restarting
        the streamer even if a StopIteration is raised.

        Parameters
        ----------
        max_iter : None or int > 0
            Maximum number of iterations to yield.
            If `None`, iterate indefinitely.

        Yields
        ------
        obj : Objects yielded by the streamer provided on init.
        '''

        count = 0
        while True:
            for obj in self.iterate():
                count += 1
                if max_iter is not None and count > max_iter:
                    return
                yield obj

Response: Iterate from the streamer infinitely.


Instruction:
Write a Python 3 script for
calculating motif enrichment metrics.

Response:
def calc_stats_iterator(motifs, fg_file, bg_file, genome=None, stats=None, ncpus=None):
    """Calculate motif enrichment metrics.

    Parameters
    ----------
    motifs : str, list or Motif instance
        A file with motifs in pwm format, a list of Motif instances or a
        single Motif instance.

    fg_file : str
        Filename of a FASTA, BED or region file with positive sequences.

    bg_file : str
        Filename of a FASTA, BED or region file with negative sequences.

    genome : str, optional
        Genome or index directory in case of BED/regions.

    stats : list, optional
        Names of metrics to calculate. See gimmemotifs.rocmetrics.__all__
        for available metrics.

    ncpus : int, optional
        Number of cores to use.

    Returns
    -------
    result : dict
        Dictionary with results where keys are motif ids and the values are
        dictionary with metric name and value pairs.
    """
    if not stats:
        stats = rocmetrics.__all__

    if isinstance(motifs, Motif):
        all_motifs = [motifs]
    else:
        if type([]) == type(motifs):
            all_motifs = motifs
        else:
            all_motifs = read_motifs(motifs, fmt="pwm")

    if ncpus is None:
        ncpus = int(MotifConfig().get_default_params()["ncpus"])
    chunksize = 240

    for i in range(0, len(all_motifs), chunksize):
        result = {}
        logger.debug("chunk %s of %s",
            (i / chunksize) + 1, len(all_motifs) // chunksize + 1)
        motifs = all_motifs[i:i + chunksize]

        fg_total = scan_to_best_match(fg_file, motifs, ncpus=ncpus, genome=genome)
        bg_total = scan_to_best_match(bg_file, motifs, ncpus=ncpus, genome=genome)

        logger.debug("calculating statistics")

        if ncpus == 1:
            it = _single_stats(motifs, stats, fg_total, bg_total)
        else:
            it = _mp_stats(motifs, stats, fg_total, bg_total, ncpus)

        for motif_id, s, ret in it:
            if motif_id not in result:
                result[motif_id] = {}
            result[motif_id][s] = ret
        yield result


Instruction:
Can you generate a brief explanation for the following Python 3 code
def calc_stats(motifs, fg_file, bg_file, genome=None, stats=None, ncpus=None):
    result = {}
    for batch_result in calc_stats_iterator(motifs, fg_file, bg_file, genome=genome, stats=stats, ncpus=ncpus):
        for motif_id in batch_result:
            if motif_id not in result:
                result[motif_id] = {}
            for s,ret in batch_result[motif_id].items():
                result[motif_id][s] = ret
    return result

Response: Calculate motif enrichment metrics.


Instruction:
Explain what the following Python 3 code does
def rank_motifs(stats, metrics=("roc_auc", "recall_at_fdr")):
    rank = {}
    combined_metrics = []
    motif_ids = stats.keys()
    background = list(stats.values())[0].keys()
    for metric in metrics:
        mean_metric_stats = [np.mean(
            [stats[m][bg][metric] for bg in background]) for m in motif_ids]
        ranked_metric_stats = rankdata(mean_metric_stats)
        combined_metrics.append(ranked_metric_stats)

    for motif, val in zip(motif_ids, np.mean(combined_metrics, 0)):
        rank[motif] = val

    return rank

Response: Determine mean rank of motifs based on metrics.


Instruction:
Implement a Python 3 function for
writing motif statistics to text file

Response:
def write_stats(stats, fname, header=None):
    """write motif statistics to text file."""
    # Write stats output to file

    for bg in list(stats.values())[0].keys():
        f = open(fname.format(bg), "w")
        if header:
            f.write(header)

        stat_keys = sorted(list(list(stats.values())[0].values())[0].keys())
        f.write("{}\t{}\n".format("Motif", "\t".join(stat_keys)))

        for motif in stats:
            m_stats = stats.get(str(motif), {}).get(bg)
            if m_stats:
                f.write("{}\t{}\n".format(
                    "_".join(motif.split("_")[:-1]),
                    "\t".join([str(m_stats[k]) for k in stat_keys])
                    ))
            else:
                logger.warn("No stats for motif {0}, skipping this motif!".format(motif.id))
            #motifs.remove(motif)
        f.close()

    return


Instruction:
Explain what the following Python 3 code does
def get_roc_values(motif, fg_file, bg_file):
    #print(calc_stats(motif, fg_file, bg_file, stats=["roc_values"], ncpus=1))
    #["roc_values"])

    try:
#        fg_result = motif.pwm_scan_score(Fasta(fg_file), cutoff=0.0, nreport=1)
#        fg_vals = [sorted(x)[-1] for x in fg_result.values()]
#
#        bg_result = motif.pwm_scan_score(Fasta(bg_file), cutoff=0.0, nreport=1)
#        bg_vals = [sorted(x)[-1] for x in bg_result.values()]

#        (x, y) = roc_values(fg_vals, bg_vals)
        stats = calc_stats(motif, fg_file, bg_file, stats=["roc_values"], ncpus=1)
        (x,y) = list(stats.values())[0]["roc_values"]
        return None,x,y
    except Exception as e:
        print(motif)
        print(motif.id)
        raise
        error = e
        return error,[],[]

Response: Calculate ROC AUC values for ROC plots.


Instruction:
How would you code a function in Python 3 to
create ROC plots for all motifs.

Response:
def create_roc_plots(pwmfile, fgfa, background, outdir):
    """Make ROC plots for all motifs."""
    motifs = read_motifs(pwmfile, fmt="pwm", as_dict=True)
    ncpus = int(MotifConfig().get_default_params()['ncpus'])
    pool = Pool(processes=ncpus)
    jobs = {}
    for bg,fname in background.items():
        for m_id, m in motifs.items():

            k = "{}_{}".format(str(m), bg)
            jobs[k] = pool.apply_async(
                                            get_roc_values,
                                            (motifs[m_id], fgfa, fname,)
                                            )
    imgdir = os.path.join(outdir, "images")
    if not os.path.exists(imgdir):
        os.mkdir(imgdir)

    roc_img_file = os.path.join(outdir, "images", "{}_roc.{}.png")

    for motif in motifs.values():
        for bg in background:
            k = "{}_{}".format(str(motif), bg)
            error, x, y = jobs[k].get()
            if error:
                logger.error("Error in thread: %s", error)
                logger.error("Motif: %s", motif)
                sys.exit(1)
            roc_plot(roc_img_file.format(motif.id, bg), x, y)


Instruction:
Create a Python 3 function to
create text report of motifs with statistics and database match.

Response:
def _create_text_report(inputfile, motifs, closest_match, stats, outdir):
    """Create text report of motifs with statistics and database match."""
    my_stats = {}
    for motif in motifs:
        match = closest_match[motif.id]
        my_stats[str(motif)] = {}
        for bg in list(stats.values())[0].keys():
            if str(motif) not in stats:
                logger.error("####")
                logger.error("{} not found".format(str(motif)))
                for s in sorted(stats.keys()):
                    logger.error(s)
                logger.error("####")
            else:
                my_stats[str(motif)][bg] = stats[str(motif)][bg].copy()
                my_stats[str(motif)][bg]["best_match"] = "_".join(match[0].split("_")[:-1])
                my_stats[str(motif)][bg]["best_match_pvalue"] = match[1][-1]

    header = ("# GimmeMotifs version {}\n"
             "# Inputfile: {}\n"
             ).format(__version__, inputfile)

    write_stats(my_stats, os.path.join(outdir, "stats.{}.txt"), header=header)


Instruction:
Write a Python 3 script to
create a graphical report from a gimme_motifs input file pwm and background.

Response:
def _create_graphical_report(inputfile, pwm, background, closest_match, outdir, stats, best_id=None):
    """Create main gimme_motifs output html report."""
    if best_id is None:
        best_id = {}

    logger.debug("Creating graphical report")

    class ReportMotif(object):
        """Placeholder for motif stats."""
        pass

    config = MotifConfig()

    imgdir = os.path.join(outdir, "images")
    if not os.path.exists(imgdir):
        os.mkdir(imgdir)

    motifs = read_motifs(pwm, fmt="pwm")

    roc_img_file = "%s_roc.%s"

    dbpwm = config.get_default_params()["motif_db"]
    pwmdir = config.get_motif_dir()

    dbmotifs = read_motifs(os.path.join(pwmdir, dbpwm), as_dict=True)

    report_motifs = []
    for motif in motifs:

        rm = ReportMotif()
        rm.id = motif.id
        rm.id_href = {"href": "#%s" % motif.id}
        rm.id_name = {"name": motif.id}
        rm.img = {"src":  os.path.join("images", "%s.png" % motif.id)}
        motif.to_img(os.path.join(outdir, "images/{}.png".format(motif.id)), fmt="PNG")

        # TODO: fix best ID
        rm.best = "Gimme"#best_id[motif.id]

        rm.consensus = motif.to_consensus()
        rm.stars = int(np.mean(
                [stats[str(motif)][bg].get("stars", 0) for bg in background]
                ) + 0.5)

        rm.bg = {}
        for bg in background:
            rm.bg[bg] = {}
            this_stats = stats.get(str(motif), {}).get(bg)
            # TODO: fix these stats
            rm.bg[bg]["e"] = "%0.2f" % this_stats.get("enr_at_fpr", 1.0)
            rm.bg[bg]["p"] = "%0.2f" % this_stats.get("phyper_at_fpr", 1.0)
            rm.bg[bg]["auc"] = "%0.3f" % this_stats.get("roc_auc", 0.5)
            rm.bg[bg]["mncp"] = "%0.3f" % this_stats.get("mncp", 1.0)
            rm.bg[bg]["roc_img"] = {"src": "images/" + os.path.basename(roc_img_file % (motif.id, bg)) + ".png"}
            rm.bg[bg][u"roc_img_link"] = {u"href": "images/" + os.path.basename(roc_img_file % (motif.id, bg)) + ".png"}

        rm.histogram_img = {"data":"images/%s_histogram.svg" % motif.id}
        rm.histogram_link= {"href":"images/%s_histogram.svg" % motif.id}

        match_id = closest_match[motif.id][0]
        dbmotifs[match_id].to_img(os.path.join(outdir, "images/{}.png".format(match_id)), fmt="PNG")

        rm.match_img = {"src":  "images/{}.png".format(match_id)}
        rm.match_id = closest_match[motif.id][0]
        rm.match_pval = "%0.2e" % closest_match[motif.id][1][-1]

        report_motifs.append(rm)

    total_report = os.path.join(outdir, "motif_report.html")

    star_img = os.path.join(config.get_template_dir(), "star.png")
    shutil.copyfile(star_img, os.path.join(outdir, "images", "star.png"))

    env = jinja2.Environment(loader=jinja2.FileSystemLoader([config.get_template_dir()]))
    template = env.get_template("report_template.jinja.html")
    # TODO: title
    result = template.render(
                    motifs=report_motifs,
                    inputfile=inputfile,
                    date=datetime.today().strftime("%d/%m/%Y"),
                    version=__version__,
                    bg_types=list(background.keys()))

    with open(total_report, "wb") as f:
        f.write(result.encode('utf-8'))


Instruction:
Here you have a function in Python 3, explain what it does
def create_denovo_motif_report(inputfile, pwmfile, fgfa, background, locfa, outdir, params, stats=None):
    logger.info("creating reports")

    motifs = read_motifs(pwmfile, fmt="pwm")

    # ROC plots
    create_roc_plots(pwmfile, fgfa, background, outdir)

    # Closest match in database
    mc = MotifComparer()
    closest_match = mc.get_closest_match(motifs)

    if stats is None:
        stats = {}
        for bg, bgfa in background.items():
            for m, s in calc_stats(motifs, fgfa, bgfa).items():
                if m not in stats:
                    stats[m] = {}
                stats[m][bg] = s

    stats = add_star(stats)

    if not params:
        params = {}
    cutoff_fpr = params.get('cutoff_fpr', 0.9)
    lwidth = np.median([len(seq) for seq in Fasta(locfa).seqs])

    # Location plots
    logger.debug("Creating localization plots")
    for motif in motifs:
        logger.debug("  {} {}".format(motif.id, motif))
        outfile = os.path.join(outdir, "images/{}_histogram.svg".format(motif.id))
        motif_localization(locfa, motif, lwidth, outfile, cutoff=cutoff_fpr)

    # Create reports
    _create_text_report(inputfile, motifs, closest_match, stats, outdir)
    _create_graphical_report(inputfile, pwmfile, background, closest_match, outdir, stats)

Response: Create a single motif report.


Instruction:
Explain what the following Python 3 code does
def axes_off(ax):
    ax.set_frame_on(False)
    ax.axes.get_yaxis().set_visible(False)
    ax.axes.get_xaxis().set_visible(False)

Response: Get rid of all axis ticks, lines, etc.


Instruction:
Write a Python 3 function for
plotting list of motifs with database match and p - value

Response:
def match_plot(plotdata, outfile):
    """Plot list of motifs with database match and p-value
    "param plotdata: list of (motif, dbmotif, pval)
    """
    fig_h = 2
    fig_w = 7

    nrows = len(plotdata)
    ncols = 2
    fig = plt.figure(figsize=(fig_w, nrows * fig_h))

    for i, (motif, dbmotif, pval) in enumerate(plotdata):
        text = "Motif: %s\nBest match: %s\np-value: %0.2e" % (motif.id, dbmotif.id, pval)

        grid = ImageGrid(fig, (nrows, ncols, i * 2 + 1),
                         nrows_ncols = (2,1),
                         axes_pad=0,
                         )

        for j in range(2):
            axes_off(grid[j])

        tmp = NamedTemporaryFile(dir=mytmpdir(), suffix=".png")
        motif.to_img(tmp.name, fmt="PNG", height=6)
        grid[0].imshow(plt.imread(tmp.name), interpolation="none")
        tmp = NamedTemporaryFile(dir=mytmpdir(), suffix=".png")
        dbmotif.to_img(tmp.name, fmt="PNG")
        grid[1].imshow(plt.imread(tmp.name), interpolation="none")

        ax = plt.subplot(nrows, ncols, i * 2 + 2)
        axes_off(ax)

        ax.text(0, 0.5, text,
        horizontalalignment='left',
        verticalalignment='center')

    plt.savefig(outfile, dpi=300, bbox_inches='tight')
    plt.close(fig)


Instruction:
Write a Python 3 script to
plot a phylogenetic tree

Response:
def motif_tree_plot(outfile, tree, data, circle=True, vmin=None, vmax=None, dpi=300):
    """
    Plot a "phylogenetic" tree
    """
    try:
        from ete3 import Tree, faces, AttrFace, TreeStyle, NodeStyle
    except ImportError:
        print("Please install ete3 to use this functionality")
        sys.exit(1)

    # Define the tree
    t, ts = _get_motif_tree(tree, data, circle, vmin, vmax)

    # Save image
    t.render(outfile, tree_style=ts, w=100, dpi=dpi, units="mm");

    # Remove the bottom (empty) half of the figure
    if circle:
        img = Image.open(outfile)
        size = img.size[0]
        spacer = 50
        img.crop((0,0,size,size/2 + spacer)).save(outfile)


Instruction:
Given the following Python 3 function, write the documentation
def check_bed_file(fname):
    if not os.path.exists(fname):
        logger.error("Inputfile %s does not exist!", fname)
        sys.exit(1)

    for i, line in enumerate(open(fname)):
        if line.startswith("#") or line.startswith("track") or line.startswith("browser"):
            # comment or BED specific stuff
            pass
        else:
            vals = line.strip().split("\t")
            if len(vals) < 3:
                logger.error("Expecting tab-seperated values (chromosome<tab>start<tab>end) on line %s of file %s", i + 1, fname)
                sys.exit(1)
            try:
                start, end = int(vals[1]), int(vals[2])
            except ValueError:
                logger.error("No valid integer coordinates on line %s of file %s", i + 1, fname)
                sys.exit(1)
            if len(vals) > 3:
                try:
                    float(vals[3])
                except ValueError:
                    pass

Response: Check if the inputfile is a valid BED - file


Instruction:
Can you generate a brief explanation for the following Python 3 code
def check_denovo_input(inputfile, params):
    background = params["background"]

    input_type = determine_file_type(inputfile)

    if input_type == "fasta":
        valid_bg = FA_VALID_BGS
    elif input_type in ["bed", "narrowpeak"]:
        genome = params["genome"]
        valid_bg = BED_VALID_BGS
        if "genomic" in background or "gc" in background:
            Genome(genome)
        # is it a valid bed-file etc.
        check_bed_file(inputfile)    # bed-specific, will also work for narrowPeak
    else:
        sys.stderr.write("Format of inputfile {} not recognized.\n".format(inputfile))
        sys.stderr.write("Input should be FASTA, BED or narrowPeak.\n")
        sys.stderr.write("See https://genome.ucsc.edu/FAQ/FAQformat.html for specifications.\n")
        sys.exit(1)

    for bg in background:
        if not bg in valid_bg:
            logger.info("Input type is %s, ignoring background type '%s'",
                            input_type, bg)
        background = [bg for bg in background if bg in valid_bg]

    if len(background) == 0:
        logger.error("No valid backgrounds specified!")
        sys.exit(1)

    return input_type, background

Response: Check if an input file is valid and return the input type and backgrounds.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def scan_to_best_match(fname, motifs, ncpus=None, genome=None, score=False):
    # Initialize scanner
    s = Scanner(ncpus=ncpus)
    s.set_motifs(motifs)
    s.set_threshold(threshold=0.0)
    if genome:
        s.set_genome(genome)

    if isinstance(motifs, six.string_types):
        motifs = read_motifs(motifs)

    logger.debug("scanning %s...", fname)
    result = dict([(m.id, []) for m in motifs])
    if score:
        it = s.best_score(fname)
    else:
        it = s.best_match(fname)
    for scores in it:
        for motif,score in zip(motifs, scores):
            result[motif.id].append(score)

    # Close the pool and reclaim memory
    del s

    return result

Response: Scan a FASTA file with motifs and return a dictionary with the best match per motif.


Instruction:
Implement a Python 3 function for
setting the background for FPR and z - score calculations.

Response:
def set_background(self, fname=None, genome=None, length=200, nseq=10000):
        """Set the background to use for FPR and z-score calculations.

        Background can be specified either as a genome name or as the
        name of a FASTA file.

        Parameters
        ----------
        fname : str, optional
            Name of FASTA file to use as background.

        genome : str, optional
            Name of genome to use to retrieve random sequences.

        length : int, optional
            Length of genomic sequences to retrieve. The default
            is 200.

        nseq : int, optional
            Number of genomic sequences to retrieve.
        """
        length = int(length)

        if genome and fname:
            raise ValueError("Need either genome or filename for background.")

        if fname:
            if not os.path.exists(fname):
                raise IOError("Background file {} does not exist!".format(fname))

            self.background = Fasta(fname)
            self.background_hash = file_checksum(fname)
            return

        if not genome:
            if self.genome:
                genome = self.genome
                logger.info("Using default background: genome {} with length {}".format(
                    genome, length))
            else:
                raise ValueError("Need either genome or filename for background.")

        logger.info("Using background: genome {} with length {}".format(genome, length))
        with Cache(CACHE_DIR) as cache:
            self.background_hash = "{}\{}".format(genome, int(length))
            fa = cache.get(self.background_hash)
            if not fa:
                fa = RandomGenomicFasta(genome, length, nseq)
                cache.set(self.background_hash, fa)
        self.background = fa


Instruction:
Implement a Python 3 function for
setting motif scanning threshold based on background sequences.

Response:
def set_threshold(self, fpr=None, threshold=None):
        """Set motif scanning threshold based on background sequences.

        Parameters
        ----------
        fpr : float, optional
            Desired FPR, between 0.0 and 1.0.

        threshold : float or str, optional
            Desired motif threshold, expressed as the fraction of the
            difference between minimum and maximum score of the PWM.
            Should either be a float between 0.0 and 1.0 or a filename
            with thresholds as created by 'gimme threshold'.

        """
        if threshold and fpr:
            raise ValueError("Need either fpr or threshold.")

        if fpr:
            fpr = float(fpr)
            if not (0.0 < fpr < 1.0):
                raise ValueError("Parameter fpr should be between 0 and 1")

        if not self.motifs:
            raise ValueError("please run set_motifs() first")

        thresholds = {}
        motifs = read_motifs(self.motifs)

        if threshold is not None:
            self.threshold = parse_threshold_values(self.motifs, threshold)
            return

        if not self.background:
            try:
                self.set_background()
            except:
                raise ValueError("please run set_background() first")

        seqs = self.background.seqs

        with Cache(CACHE_DIR) as cache:
            scan_motifs = []
            for motif in motifs:
                k = "{}|{}|{:.4f}".format(motif.hash(), self.background_hash, fpr)

                threshold = cache.get(k)
                if threshold is None:
                    scan_motifs.append(motif)
                else:
                    if np.isclose(threshold, motif.pwm_max_score()):
                        thresholds[motif.id] = None
                    elif np.isclose(threshold, motif.pwm_min_score()):
                        thresholds[motif.id] = 0.0
                    else:
                        thresholds[motif.id] = threshold

            if len(scan_motifs) > 0:
                logger.info("Determining FPR-based threshold")
                for motif, threshold in self._threshold_from_seqs(scan_motifs, seqs, fpr):
                    k = "{}|{}|{:.4f}".format(motif.hash(), self.background_hash, fpr)
                    cache.set(k, threshold)
                    if np.isclose(threshold, motif.pwm_max_score()):
                        thresholds[motif.id] = None
                    elif np.isclose(threshold, motif.pwm_min_score()):
                        thresholds[motif.id] = 0.0
                    else:
                        thresholds[motif.id] = threshold
        self.threshold_str = "{}_{}_{}".format(fpr, threshold, self.background_hash)
        self.threshold = thresholds


Instruction:
Create a Python 3 function to
count the number of matches above the cutoff returns an iterator of lists containing integer counts

Response:
def count(self, seqs, nreport=100, scan_rc=True):
        """
        count the number of matches above the cutoff
        returns an iterator of lists containing integer counts
        """
        for matches in self.scan(seqs, nreport, scan_rc):
            counts = [len(m) for m in matches]
            yield counts


Instruction:
Can you create a Python 3 function that
counts the number of matches above the cutoff returns an iterator of lists containing integer counts

Response:
def total_count(self, seqs, nreport=100, scan_rc=True):
        """
        count the number of matches above the cutoff
        returns an iterator of lists containing integer counts
        """

        count_table = [counts for counts in self.count(seqs, nreport, scan_rc)]
        return np.sum(np.array(count_table), 0)


Instruction:
Write a Python 3 function for
returning an iterator of lists containing the scores of the best match of each motif in each sequence

Response:
def best_score(self, seqs, scan_rc=True, normalize=False):
        """
        give the score of the best match of each motif in each sequence
        returns an iterator of lists containing floats
        """
        self.set_threshold(threshold=0.0)
        if normalize and len(self.meanstd) == 0:
            self.set_meanstd()
            means = np.array([self.meanstd[m][0] for m in self.motif_ids])
            stds = np.array([self.meanstd[m][1] for m in self.motif_ids])

        for matches in self.scan(seqs, 1, scan_rc):
            scores = np.array([sorted(m, key=lambda x: x[0])[0][0] for m in matches if len(m) > 0])
            if normalize:
                scores = (scores - means) / stds
            yield scores


Instruction:
Can you tell what is the following Python 3 function doing
def best_match(self, seqs, scan_rc=True):
        self.set_threshold(threshold=0.0)
        for matches in self.scan(seqs, 1, scan_rc):
            yield [m[0] for m in matches]

Response: gives the best match of each motif in each sequence


Instruction:
Given the following Python 3 function, write the documentation
def scan(self, seqs, nreport=100, scan_rc=True, normalize=False):

        if not self.threshold:
            sys.stderr.write(
                "Using default threshold of 0.95. "
                "This is likely not optimal!\n"
                )
            self.set_threshold(threshold=0.95)

        seqs = as_fasta(seqs, genome=self.genome)

        it = self._scan_sequences(seqs.seqs,
                    nreport, scan_rc)

        if normalize:
            if len(self.meanstd) == 0:
                self.set_meanstd()
            mean_std = [self.meanstd.get(m_id) for m_id in self.motif_ids]
            means = [x[0] for x in  mean_std]
            stds = [x[1] for x in  mean_std]

        for result in it:
            if normalize:
                zresult = []
                for i,mrow in enumerate(result):
                    mrow = [((x[0] - means[i]) / stds[i], x[1], x[2]) for x in mrow]
                    zresult.append(mrow)
                yield zresult
            else:
                yield result

Response: Scan a set of regions and sequences and yield motifs.


Instruction:
Given the following Python 3 function, write the documentation
def roc(args):
    outputfile = args.outfile
    # Default extension for image
    if outputfile and not outputfile.endswith(".png"):
        outputfile += ".png"

    motifs = read_motifs(args.pwmfile, fmt="pwm")

    ids = []
    if args.ids:
        ids = args.ids.split(",")
    else:
        ids = [m.id for m in motifs]
    motifs = [m for m in motifs if (m.id in ids)]

    stats = [
            "phyper_at_fpr",
            "roc_auc",
            "pr_auc",
            "enr_at_fpr",
            "recall_at_fdr",
            "roc_values",
            "matches_at_fpr",
            ]

    plot_x = []
    plot_y = []
    legend = []

    f_out = sys.stdout
    if args.outdir:
        if not os.path.exists(args.outdir):
            os.makedirs(args.outdir)
        f_out = open(args.outdir + "/gimme.roc.report.txt", "w")

    # Print the metrics
    f_out.write("Motif\t# matches\t# matches background\tP-value\tlog10 P-value\tROC AUC\tPR AUC\tEnr. at 1% FPR\tRecall at 10% FDR\n")

    for motif_stats in calc_stats_iterator(motifs, args.sample, args.background,
            genome=args.genome, stats=stats, ncpus=args.ncpus):

        for motif in motifs:
            if str(motif) in motif_stats:
                if outputfile:
                    x, y = motif_stats[str(motif)]["roc_values"]
                    plot_x.append(x)
                    plot_y.append(y)
                    legend.append(motif.id)
                log_pvalue = np.inf
                if motif_stats[str(motif)]["phyper_at_fpr"] > 0:
                    log_pvalue = -np.log10(motif_stats[str(motif)]["phyper_at_fpr"])
                f_out.write("{}\t{:d}\t{:d}\t{:.2e}\t{:.3f}\t{:.3f}\t{:.3f}\t{:.2f}\t{:0.4f}\n".format(
                      motif.id,
                      motif_stats[str(motif)]["matches_at_fpr"][0],
                      motif_stats[str(motif)]["matches_at_fpr"][1],
                      motif_stats[str(motif)]["phyper_at_fpr"],
                      log_pvalue,
                      motif_stats[str(motif)]["roc_auc"],
                      motif_stats[str(motif)]["pr_auc"],
                      motif_stats[str(motif)]["enr_at_fpr"],
                      motif_stats[str(motif)]["recall_at_fdr"],
                      ))
    f_out.close()

    if args.outdir:
        html_report(
            args.outdir,
            args.outdir + "/gimme.roc.report.txt",
            args.pwmfile,
            0.01,
            )

    # Plot the ROC curve
    if outputfile:
        roc_plot(outputfile, plot_x, plot_y, ids=legend)

Response: Calculate ROC_AUC and other metrics and optionally plot ROC curve.


Instruction:
Here you have a function in Python 3, explain what it does
def ssd(p1, p2):
    return 2 - np.sum([(a-b)**2 for a,b in zip(p1,p2)])

Response: Calculates motif position similarity based on sum of squared distances.


Instruction:
Make a summary of the following Python 3 code
def seqcor(m1, m2, seq=None):
    l1 = len(m1)
    l2 = len(m2)

    l = max(l1, l2)

    if seq is None:
        seq = RCDB

    L = len(seq)

    # Scan RC de Bruijn sequence
    result1 = pfmscan(seq, m1.pwm, m1.pwm_min_score(), len(seq), False, True)
    result2 = pfmscan(seq, m2.pwm, m2.pwm_min_score(), len(seq), False, True)

    # Reverse complement of motif 2
    result3 = pfmscan(seq, m2.rc().pwm, m2.rc().pwm_min_score(), len(seq), False, True)

    result1 = np.array(result1)
    result2 = np.array(result2)
    result3 = np.array(result3)

    # Return maximum correlation
    c = []
    for i in range(l1 - l1 // 3):
        c.append([1 - distance.correlation(result1[:L-l-i],result2[i:L-l]), i, 1])
        c.append([1 - distance.correlation(result1[:L-l-i],result3[i:L-l]), i, -1])
    for i in range(l2 - l2 // 3):
        c.append([1 - distance.correlation(result1[i:L-l],result2[:L-l-i]), -i, 1])
        c.append([1 - distance.correlation(result1[i:L-l],result3[:L-l-i]), -i, -1])

    return sorted(c, key=lambda x: x[0])[-1]

Response: Calculates similarity based on Pearson correlation of scores of two Motif objects.


Instruction:
Can you tell what is the following Python 3 function doing
def compare_motifs(self, m1, m2, match="total", metric="wic", combine="mean", pval=False):
        if metric == "seqcor":
            return seqcor(m1, m2)
        elif match == "partial":
            if pval:
                return self.pvalue(m1, m2, "total", metric, combine, self.max_partial(m1.pwm, m2.pwm, metric, combine))
            elif metric in ["pcc", "ed", "distance", "wic", "chisq", "ssd"]:
                return self.max_partial(m1.pwm, m2.pwm, metric, combine)
            else:
                return self.max_partial(m1.pfm, m2.pfm, metric, combine)

        elif match == "total":
            if pval:
                return self.pvalue(m1, m2, match, metric, combine, self.max_total(m1.pwm, m2.pwm, metric, combine))
            elif metric in ["pcc", 'akl']:
                # Slightly randomize the weight matrix
                return self.max_total(m1.wiggle_pwm(), m2.wiggle_pwm(), metric, combine)
            elif metric in ["ed", "distance", "wic", "chisq", "pcc", "ssd"]:
                return self.max_total(m1.pwm, m2.pwm, metric, combine)
            else:
                return self.max_total(m1.pfm, m2.pfm, metric, combine)

        elif match == "subtotal":
            if metric in ["pcc", "ed", "distance", "wic", "chisq", "ssd"]:
                return self.max_subtotal(m1.pwm, m2.pwm, metric, combine)
            else:
                return self.max_subtotal(m1.pfm, m2.pfm, metric, combine)

Response:
Compare two motifs.

        The similarity metric can be any of seqcor, pcc, ed, distance, wic,
        chisq, akl or ssd. If match is 'total' the similarity score is
        calculated for the whole match, including positions that are not
        present in both motifs. If match is partial or subtotal, only the
        matching psotiions are used to calculate the score. The score of
        individual position is combined using either the mean or the sum.

        Note that the match and combine parameters have no effect on the seqcor
        similarity metric.

        Parameters
        ----------
        m1 : Motif instance
            Motif instance 1.

        m2 : Motif instance
            Motif instance 2.

        match : str, optional
            Match can be "partial", "subtotal" or "total". Not all metrics use
            this.

        metric : str, optional
            Distance metric.

        combine : str, optional
            Combine positional scores using "mean" or "sum". Not all metrics
            use this.

        pval : bool, optional
            Calculate p-vale of match.

        Returns
        -------
        score, position, strand


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_all_scores(self, motifs, dbmotifs, match, metric, combine,
                            pval=False, parallel=True, trim=None, ncpus=None):
        """Pairwise comparison of a set of motifs compared to reference motifs.

        Parameters
        ----------
        motifs : list
            List of Motif instances.

        dbmotifs : list
            List of Motif instances.

        match : str
            Match can be "partial", "subtotal" or "total". Not all metrics use
            this.

        metric : str
            Distance metric.

        combine : str
            Combine positional scores using "mean" or "sum". Not all metrics
            use this.

        pval : bool , optional
            Calculate p-vale of match.

        parallel : bool , optional
            Use multiprocessing for parallel execution. True by default.

        trim : float or None
            If a float value is specified, motifs are trimmed used this IC
            cutoff before comparison.

        ncpus : int or None
            Specifies the number of cores to use for parallel execution.

        Returns
        -------
        scores : dict
            Dictionary with scores.
        """
        # trim motifs first, if specified
        if trim:
            for m in motifs:
                m.trim(trim)
            for m in dbmotifs:
                m.trim(trim)

        # hash of result scores
        scores = {}

        if parallel:
            # Divide the job into big chunks, to keep parallel overhead to minimum
            # Number of chunks = number of processors available
            if ncpus is None:
                ncpus = int(MotifConfig().get_default_params()["ncpus"])

            pool = Pool(processes=ncpus, maxtasksperchild=1000)

            batch_len = len(dbmotifs) // ncpus
            if batch_len <= 0:
                batch_len = 1
            jobs = []
            for i in range(0, len(dbmotifs), batch_len):
                # submit jobs to the job server

                p = pool.apply_async(_get_all_scores,
                    args=(self, motifs, dbmotifs[i: i + batch_len], match, metric, combine, pval))
                jobs.append(p)

            pool.close()
            for job in jobs:
                # Get the job result
                result = job.get()
                # and update the result score
                for m1,v in result.items():
                    for m2, s in v.items():
                        if m1 not in scores:
                            scores[m1] = {}
                        scores[m1][m2] = s

            pool.join()
        else:
            # Do the whole thing at once if we don't want parallel
            scores = _get_all_scores(self, motifs, dbmotifs, match, metric, combine, pval)

        return scores

Response: Pairwise comparison of a set of motifs compared to reference motifs.


Instruction:
Given the following Python 3 function, write the documentation
def get_closest_match(self, motifs, dbmotifs=None, match="partial", metric="wic",combine="mean", parallel=True, ncpus=None):

        if dbmotifs is None:
            pwm = self.config.get_default_params()["motif_db"]
            pwmdir = self.config.get_motif_dir()
            dbmotifs = os.path.join(pwmdir, pwm)

        motifs = parse_motifs(motifs)
        dbmotifs = parse_motifs(dbmotifs)

        dbmotif_lookup = dict([(m.id, m) for m in dbmotifs])

        scores = self.get_all_scores(motifs, dbmotifs, match, metric, combine, parallel=parallel, ncpus=ncpus)
        for motif in scores:
            scores[motif] = sorted(
                    scores[motif].items(),
                    key=lambda x:x[1][0]
                    )[-1]

        for motif in motifs:
            dbmotif, score = scores[motif.id]
            pval, pos, orient = self.compare_motifs(
                    motif, dbmotif_lookup[dbmotif], match, metric, combine, True)

            scores[motif.id] = [dbmotif, (list(score) + [pval])]

        return scores

Response: Returns a dict of motifs with the closest match.


Instruction:
Can you tell what is the following Python 3 function doing
def list_regions(service):
    for region in service.regions():
        print '%(name)s: %(endpoint)s' % {
            'name': region.name,
            'endpoint': region.endpoint,
        }

Response: List the regions for the service


Instruction:
Can you tell what is the following Python 3 function doing
def elb_table(balancers):
    t = prettytable.PrettyTable(['Name', 'DNS', 'Ports', 'Zones', 'Created'])
    t.align = 'l'
    for b in balancers:
        ports = ['%s: %s -> %s' % (l[2], l[0], l[1]) for l in b.listeners]
        ports = '\n'.join(ports)
        zones = '\n'.join(b.availability_zones)
        t.add_row([b.name, b.dns_name, ports, zones, b.created_time])
    return t

Response: Print nice looking table of information from list of load balancers


Instruction:
Write a Python 3 function for
printing nice looking table of information from list of instances

Response:
def ec2_table(instances):
    """
    Print nice looking table of information from list of instances
    """
    t = prettytable.PrettyTable(['ID', 'State', 'Monitored', 'Image', 'Name', 'Type', 'SSH key', 'DNS'])
    t.align = 'l'
    for i in instances:
        name = i.tags.get('Name', '')
        t.add_row([i.id, i.state, i.monitored, i.image_id, name, i.instance_type, i.key_name, i.dns_name])
    return t


Instruction:
Implement a Python 3 function for
printing nice looking table of information from images

Response:
def ec2_image_table(images):
    """
    Print nice looking table of information from images
    """
    t = prettytable.PrettyTable(['ID', 'State', 'Name', 'Owner', 'Root device', 'Is public', 'Description'])
    t.align = 'l'
    for i in images:
        t.add_row([i.id, i.state, i.name, i.ownerId, i.root_device_type, i.is_public, i.description])
    return t


Instruction:
Can you generate the documentation for the following Python 3 function
def ec2_fab(service, args):
    instance_ids = args.instances
    instances = service.list(elb=args.elb, instance_ids=instance_ids)
    hosts = service.resolve_hosts(instances)

    fab.env.hosts = hosts
    fab.env.key_filename = settings.get('SSH', 'KEY_FILE')
    fab.env.user = settings.get('SSH', 'USER', getpass.getuser())
    fab.env.parallel = True

    fabfile = find_fabfile(args.file)
    if not fabfile:
        print 'Couldn\'t find any fabfiles!'
        return

    fab.env.real_fabile = fabfile
    docstring, callables, default = load_fabfile(fabfile)
    fab_state.commands.update(callables)

    commands_to_run = parse_arguments(args.methods)
    for name, args, kwargs, arg_hosts, arg_roles, arg_exclude_hosts in commands_to_run:
        fab.execute(name,
                    hosts=arg_hosts,
                    roles=arg_roles,
                    exclude_hosts=arg_exclude_hosts,
                    *args, **kwargs)

Response: Run Fabric commands against EC2 instances


Instruction:
Can you implement a function in Python 3 that
yields data from an input stream into one batch of size buffer_size.

Response:
def buffer_stream(stream, buffer_size, partial=False, axis=None):
    '''Buffer "data" from an stream into one data object.

    Parameters
    ----------
    stream : stream
        The stream to buffer

    buffer_size : int > 0
        The number of examples to retain per batch.

    partial : bool, default=False
        If True, yield a final partial batch on under-run.

    axis : int or None
        If `None` (default), concatenate data along a new 0th axis.
        Otherwise, concatenation is performed along the specified axis.

        This is primarily useful when combining data that already has a
        dimension for buffer index, e.g., when buffering buffers.

    Yields
    ------
    batch
        A batch of size at most `buffer_size`

    Raises
    ------
    DataError
        If the stream contains items that are not data-like.
    '''

    data = []
    count = 0

    for item in stream:
        data.append(item)
        count += 1

        if count < buffer_size:
            continue
        try:
            yield __stack_data(data, axis=axis)
        except (TypeError, AttributeError):
            raise DataError("Malformed data stream: {}".format(data))
        finally:
            data = []
            count = 0

    if data and partial:
        yield __stack_data(data, axis=axis)


Instruction:
Can you create a Python 3 function that
reformats data as tuples.

Response:
def tuples(stream, *keys):
    """Reformat data as tuples.

    Parameters
    ----------
    stream : iterable
        Stream of data objects.

    *keys : strings
        Keys to use for ordering data.

    Yields
    ------
    items : tuple of np.ndarrays
        Data object reformated as a tuple.

    Raises
    ------
    DataError
        If the stream contains items that are not data-like.
    KeyError
        If a data object does not contain the requested key.
    """
    if not keys:
        raise PescadorError('Unable to generate tuples from '
                            'an empty item set')
    for data in stream:
        try:
            yield tuple(data[key] for key in keys)
        except TypeError:
            raise DataError("Malformed data stream: {}".format(data))


Instruction:
How would you code a function in Python 3 to
reformat data objects as keras - compatible tuples.

Response:
def keras_tuples(stream, inputs=None, outputs=None):
    """Reformat data objects as keras-compatible tuples.

    For more detail: https://keras.io/models/model/#fit

    Parameters
    ----------
    stream : iterable
        Stream of data objects.

    inputs : string or iterable of strings, None
        Keys to use for ordered input data.
        If not specified, returns `None` in its place.

    outputs : string or iterable of strings, default=None
        Keys to use for ordered output data.
        If not specified, returns `None` in its place.

    Yields
    ------
    x : np.ndarray, list of np.ndarray, or None
        If `inputs` is a string, `x` is a single np.ndarray.
        If `inputs` is an iterable of strings, `x` is a list of np.ndarrays.
        If `inputs` is a null type, `x` is None.
    y : np.ndarray, list of np.ndarray, or None
        If `outputs` is a string, `y` is a single np.ndarray.
        If `outputs` is an iterable of strings, `y` is a list of np.ndarrays.
        If `outputs` is a null type, `y` is None.

    Raises
    ------
    DataError
        If the stream contains items that are not data-like.
    """
    flatten_inputs, flatten_outputs = False, False
    if inputs and isinstance(inputs, six.string_types):
        inputs = [inputs]
        flatten_inputs = True
    if outputs and isinstance(outputs, six.string_types):
        outputs = [outputs]
        flatten_outputs = True

    inputs, outputs = (inputs or []), (outputs or [])
    if not inputs + outputs:
        raise PescadorError('At least one key must be given for '
                            '`inputs` or `outputs`')

    for data in stream:
        try:
            x = list(data[key] for key in inputs) or None
            if len(inputs) == 1 and flatten_inputs:
                x = x[0]

            y = list(data[key] for key in outputs) or None
            if len(outputs) == 1 and flatten_outputs:
                y = y[0]

            yield (x, y)
        except TypeError:
            raise DataError("Malformed data stream: {}".format(data))


Instruction:
How would you implement a function in Python 3 that
creates a histogram of motif location.

Response:
def location(args):
    """
    Creates histrogram of motif location.

    Parameters
    ----------
    args : argparse object
        Command line arguments.
    """
    fastafile = args.fastafile
    pwmfile = args.pwmfile

    lwidth = args.width
    if not lwidth:
        f = Fasta(fastafile)
        lwidth = len(f.items()[0][1])
        f = None

    jobs = []
    motifs = pwmfile_to_motifs(pwmfile)
    ids = [motif.id for motif in motifs]
    if args.ids:
        ids = args.ids.split(",")

    n_cpus = int(MotifConfig().get_default_params()["ncpus"])
    pool = Pool(processes=n_cpus, maxtasksperchild=1000)
    for motif in motifs:
        if motif.id in ids:
            outfile = os.path.join("%s_histogram" % motif.id)
            jobs.append(
                    pool.apply_async(
                        motif_localization,
                        (fastafile,motif,lwidth,outfile, args.cutoff)
                        ))

    for job in jobs:
        job.get()


Instruction:
Create a Python 3 function to
find location of executable.

Response:
def which(fname):
    """Find location of executable."""
    if "PATH" not in os.environ or not os.environ["PATH"]:
        path = os.defpath
    else:
        path = os.environ["PATH"]

    for p in [fname] + [os.path.join(x, fname) for x in path.split(os.pathsep)]:
        p = os.path.abspath(p)
        if os.access(p, os.X_OK) and not os.path.isdir(p):
            return p

    p = sp.Popen("locate %s" % fname, shell=True, stdout=sp.PIPE, stderr=sp.PIPE)
    (stdout, stderr) = p.communicate()
    if not stderr:
        for p in stdout.decode().split("\n"):
            if (os.path.basename(p) == fname) and (
                os.access(p, os.X_OK)) and (
                not os.path.isdir(p)):
                return p


Instruction:
Write a Python 3 function that can
find all files in a directory by extension.

Response:
def find_by_ext(dirname, ext):
    """Find all files in a directory by extension."""
    # Get all fasta-files
    try:
        files = os.listdir(dirname)
    except OSError:
        if os.path.exists(dirname):
            cmd = "find {0} -maxdepth 1 -name \"*\"".format(dirname)
            p = sp.Popen(cmd, shell=True, stdout=sp.PIPE, stderr=sp.PIPE)
            stdout, _stderr = p.communicate()
            files = [os.path.basename(fname) for fname in stdout.decode().splitlines()]
        else:
            raise

    retfiles = [os.path.join(dirname, fname) for fname in files if
                    os.path.splitext(fname)[-1] in ext]

    return retfiles


Instruction:
How would you implement a function in Python 3 that
returns list of Motif instances from default motif database.

Response:
def default_motifs():
    """Return list of Motif instances from default motif database."""
    config = MotifConfig()
    d = config.get_motif_dir()
    m = config.get_default_params()['motif_db']

    if not d or not m:
        raise ValueError("default motif database not configured")

    fname = os.path.join(d, m)
    with open(fname) as f:
        motifs = read_motifs(f)

    return motifs


Instruction:
How would you explain what the following Python 3 function does
def motif_from_align(align):
    width = len(align[0])
    nucs = {"A":0,"C":1,"G":2,"T":3}
    pfm =  [[0 for _ in range(4)] for _ in range(width)]
    for row in align:
        for i in range(len(row)):
            pfm[i][nucs[row[i]]] += 1
    m = Motif(pfm)
    m.align = align[:]
    return m

Response: Convert an alignment to a motif.


Instruction:
Write a Python 3 script for
converting a consensus sequence to a motif.

Response:
def motif_from_consensus(cons, n=12):
    """Convert consensus sequence to motif.

    Converts a consensus sequences using the nucleotide IUPAC alphabet to a
    motif.

    Parameters
    ----------
    cons : str
        Consensus sequence using the IUPAC alphabet.
    n : int , optional
        Count used to convert the sequence to a PFM.

    Returns
    -------
    m : Motif instance
        Motif created from the consensus.
    """
    width = len(cons)
    nucs = {"A":0,"C":1,"G":2,"T":3}
    pfm = [[0 for _ in range(4)] for _ in range(width)]
    m = Motif()
    for i,char in enumerate(cons):
        for nuc in m.iupac[char.upper()]:
            pfm[i][nucs[nuc]] = n / len(m.iupac[char.upper()])
    m = Motif(pfm)
    m.id = cons
    return m


Instruction:
Implement a function in Python 3 to
parse motifs in a variety of formats to return a list of motifs.

Response:
def parse_motifs(motifs):
    """Parse motifs in a variety of formats to return a list of motifs.

    Parameters
    ----------

    motifs : list or str
        Filename of motif,  list of motifs or single Motif instance.

    Returns
    -------

    motifs : list
        List of Motif instances.
    """
    if isinstance(motifs, six.string_types):
        with open(motifs) as f:
            if motifs.endswith("pwm") or motifs.endswith("pfm"):
                motifs = read_motifs(f, fmt="pwm")
            elif motifs.endswith("transfac"):
                motifs = read_motifs(f, fmt="transfac")
            else:
                motifs = read_motifs(f)
    elif isinstance(motifs, Motif):
        motifs = [motifs]
    else:
        if not isinstance(list(motifs)[0], Motif):
            raise ValueError("Not a list of motifs")

    return list(motifs)


Instruction:
Explain what the following Python 3 code does
def _read_motifs_from_filehandle(handle, fmt):
    if fmt.lower() == "pwm":
        motifs = _read_motifs_pwm(handle)
    if fmt.lower() == "transfac":
        motifs = _read_motifs_transfac(handle)
    if fmt.lower() == "xxmotif":
        motifs = _read_motifs_xxmotif(handle)
    if fmt.lower() == "align":
        motifs = _read_motifs_align(handle)
    if fmt.lower() == "jaspar":
        motifs = _read_motifs_jaspar(handle)

    if handle.name:
        base = os.path.splitext(handle.name)[0]
        map_file = base + ".motif2factors.txt"
        if os.path.exists(map_file):
            m2f_direct = {}
            m2f_indirect = {}
            for line in open(map_file):
                try:
                    motif,*factor_info = line.strip().split("\t")
                    if len(factor_info) == 1:
                        m2f_direct[motif] = factor_info[0].split(",")
                    elif len(factor_info) == 3:
                        if factor_info[2] == "Y":
                            m2f_direct[motif] = m2f_direct.get(motif, []) + [factor_info[0]]
                        else:
                            m2f_indirect[motif] = m2f_indirect.get(motif, []) + [factor_info[0]]
                except:
                    pass
            for motif in motifs:
                if motif.id in m2f_direct:
                    motif.factors[DIRECT_NAME] = m2f_direct[motif.id]
                if motif.id in m2f_indirect:
                    motif.factors[INDIRECT_NAME] = m2f_indirect[motif.id]
        for motif in motifs:
            for n in [DIRECT_NAME, INDIRECT_NAME]:
                motif.factors[n] = list(set(motif.factors[n]))
    return motifs

Response: Reads motifs from a file - like object.


Instruction:
Implement a function in Python 3 to
read motifs from a file or stream or file - like object.

Response:
def read_motifs(infile=None, fmt="pwm", as_dict=False):
    """
    Read motifs from a file or stream or file-like object.

    Parameters
    ----------
    infile : string or file-like object, optional
        Motif database, filename of motif file or file-like object. If infile
        is not specified the default motifs as specified in the config file
        will be returned.

    fmt : string, optional
        Motif format, can be 'pwm', 'transfac', 'xxmotif', 'jaspar' or 'align'.

    as_dict : boolean, optional
        Return motifs as a dictionary with motif_id, motif pairs.

    Returns
    -------
    motifs : list
        List of Motif instances. If as_dict is set to True, motifs is a
        dictionary.
    """
    if infile is None or isinstance(infile, six.string_types):
        infile = pwmfile_location(infile)
        with open(infile) as f:
            motifs = _read_motifs_from_filehandle(f, fmt)
    else:
        motifs = _read_motifs_from_filehandle(infile, fmt)

    if as_dict:
        motifs = {m.id:m for m in motifs}

    return motifs


Instruction:
Make a summary of the following Python 3 code
def information_content(self):
        ic = 0
        for row in self.pwm:
            ic += 2.0 + np.sum([row[x] * log(row[x])/log(2) for x in range(4) if row[x] > 0])
        return ic

Response: Return the total information content of the motif.


Instruction:
Write a Python 3 function that can
return the minimum PWM score.

Response:
def pwm_min_score(self):
        """Return the minimum PWM score.

        Returns
        -------
        score : float
            Minimum PWM score.
        """
        if self.min_score is None:
            score = 0
            for row in self.pwm:
                score += log(min(row) / 0.25 + 0.01)
            self.min_score = score

        return self.min_score


Instruction:
How would you explain what the following Python 3 function does
def pwm_max_score(self):
        if self.max_score is None:
            score = 0
            for row in self.pwm:
                score += log(max(row) / 0.25 + 0.01)
            self.max_score = score

        return self.max_score

Response: Return the maximum PWM score.


Instruction:
Can you write a function in Python 3 where it
calculates the log - oddds score for a specific k - mer.

Response:
def score_kmer(self, kmer):
        """Calculate the log-odds score for a specific k-mer.

        Parameters
        ----------
        kmer : str
            String representing a kmer. Should be the same length as the motif.

        Returns
        -------
        score : float
            Log-odd score.
        """
        if len(kmer) != len(self.pwm):
            raise Exception("incorrect k-mer length")

        score = 0.0
        d = {"A":0, "C":1, "G":2, "T":3}
        for nuc, row in zip(kmer.upper(), self.pwm):
            score += log(row[d[nuc]] / 0.25 + 0.01)

        return score


Instruction:
Can you create a Python 3 function that
converts a PFM with counts to a PFM with fractions.

Response:
def pfm_to_pwm(self, pfm, pseudo=0.001):
        """Convert PFM with counts to a PFM with fractions.

        Parameters
        ----------
        pfm : list
            2-dimensional list with counts.
        pseudo : float
            Pseudocount used in conversion.

        Returns
        -------
        pwm : list
            2-dimensional list with fractions.
        """
        return [[(x + pseudo)/(float(np.sum(row)) + pseudo * 4) for x in row] for row in pfm]


Instruction:
Create a Python 3 function for
returning motif formatted in MotEvo format.

Response:
def to_motevo(self):
        """Return motif formatted in MotEvo (TRANSFAC-like) format

        Returns
        -------
        m : str
            String of motif in MotEvo format.
        """
        m = "//\n"
        m += "NA {}\n".format(self.id)
        m += "P0\tA\tC\tG\tT\n"
        for i, row in enumerate(self.pfm):
            m += "{}\t{}\n".format(i, "\t".join([str(int(x)) for x in row]))
        m += "//"
        return m


Instruction:
Can you generate the documentation for the following Python 3 function
def to_transfac(self):
        m = "%s\t%s\t%s\n" % ("DE", self.id, "unknown")
        for i, (row, cons) in enumerate(zip(self.pfm, self.to_consensus())):
            m += "%i\t%s\t%s\n" % (i, "\t".join([str(int(x)) for x in row]), cons)
        m += "XX"
        return m

Response:
Return motif formatted in TRANSFAC format

        Returns ------- str str


Instruction:
Can you generate the documentation for the following Python 3 function
def to_meme(self):
        motif_id = self.id.replace(" ", "_")
        m = "MOTIF %s\n" % motif_id
        m += "BL   MOTIF %s width=0 seqs=0\n"% motif_id
        m += "letter-probability matrix: alength= 4 w= %s nsites= %s E= 0\n" % (len(self), np.sum(self.pfm[0]))
        m +="\n".join(["\t".join(["%s" % x for x in row]) for row in self.pwm])
        return m

Response:
Return motif formatted in MEME format

        Returns ------- str str
            String of motif in MEME format


Instruction:
Explain what the following Python 3 code does
def ic_pos(self, row1, row2=None):
        if row2 is None:
            row2 = [0.25,0.25,0.25,0.25]

        score = 0
        for a,b in zip(row1, row2):
            if a > 0:
                score += a * log(a / b) / log(2)
        return score

Response: Calculate the information content of one position.


Instruction:
Given the following Python 3 function, write the documentation
def pcc_pos(self, row1, row2):
        mean1 = np.mean(row1)
        mean2 = np.mean(row2)

        a = 0
        x = 0
        y = 0
        for n1, n2 in zip(row1, row2):
            a += (n1 - mean1) * (n2 - mean2)
            x += (n1 - mean1) ** 2
            y += (n2 - mean2) ** 2

        if a == 0:
            return 0
        else:
            return a / sqrt(x * y)

Response:
Calculate the Pearson correlation coefficient of one position
            compared to another position
           .


Instruction:
Given the following Python 3 function, write the documentation
def rc(self):
        m = Motif()
        m.pfm = [row[::-1] for row in self.pfm[::-1]]
        m.pwm = [row[::-1] for row in self.pwm[::-1]]
        m.id = self.id + "_revcomp"
        return m

Response: Return the reverse complemented motif.


Instruction:
Write a Python 3 function that can
remove all positions at the specified edge_ic_cutoff from the motif.

Response:
def trim(self, edge_ic_cutoff=0.4):
        """Trim positions with an information content lower than the threshold.

        The default threshold is set to 0.4. The Motif will be changed in-place.

        Parameters
        ----------
        edge_ic_cutoff : float, optional
            Information content threshold. All motif positions at the flanks
            with an information content lower thab this will be removed.

        Returns
        -------
        m : Motif instance
        """
        pwm = self.pwm[:]
        while len(pwm) > 0 and self.ic_pos(pwm[0]) < edge_ic_cutoff:
            pwm = pwm[1:]
            self.pwm = self.pwm[1:]
            self.pfm = self.pfm[1:]
        while len(pwm) > 0 and self.ic_pos(pwm[-1]) < edge_ic_cutoff:
            pwm = pwm[:-1]
            self.pwm = self.pwm[:-1]
            self.pfm = self.pfm[:-1]

        self.consensus = None
        self.min_score = None
        self.max_score = None
        self.wiggled_pwm = None

        return self


Instruction:
Implement a Python 3 function for
scanning FASTA with the motif as a consensus sequence.

Response:
def consensus_scan(self, fa):
        """Scan FASTA with the motif as a consensus sequence.

        Parameters
        ----------
        fa : Fasta object
            Fasta object to scan

        Returns
        -------
        matches : dict
            Dictionaru with matches.
        """
        regexp = "".join(["[" + "".join(self.iupac[x.upper()]) + "]" for x in self.to_consensusv2()])
        p = re.compile(regexp)
        matches = {}
        for name,seq in fa.items():
            matches[name] = []
            for match in p.finditer(seq):
                middle = (match.span()[1] + match.span()[0]) / 2
                matches[name].append(middle)
        return matches


Instruction:
Explain what the following Python 3 code does
def pwm_scan(self, fa, cutoff=0.9, nreport=50, scan_rc=True):
        c = self.pwm_min_score() + (
            self.pwm_max_score() - self.pwm_min_score()) * cutoff
        pwm = self.pwm
        matches = {}
        for name, seq in fa.items():
            matches[name] = []
            result = pfmscan(seq.upper(), pwm, c, nreport, scan_rc)
            for _,pos,_ in result:
                matches[name].append(pos)
        return matches

Response: Scan the PWM sequences with this motif.