Instruction:
How would you code a function in Python 3 to
generate k - mer counts from a list of DNA sequences

Response:
def kmer_count(seq_list, k):
    """
    Generate k-mer counts from a set of sequences

    Args:
        seq_list (iterable): List of DNA sequences (with letters from {A, C, G, T})
        k (int): K in k-mer.
    Returns:
        pandas.DataFrame: Count matrix for seach sequence in seq_list

    Example:
    >>> kmer_count(["ACGTTAT", "GACGCGA"], 2)
       AA  AC  AG  AT  CA  CC  CG  CT  GA  GC  GG  GT  TA  TC  TG  TT
    0   0   1   0   1   0   0   1   0   0   0   0   1   1   0   0   1
    1   0   1   0   0   0   0   2   0   2   1   0   0   0   0   0   0
    """
    # generate all k-mers
    all_kmers = generate_all_kmers(k)
    kmer_count_list = []
    for seq in seq_list:
        kmer_count_list.append([seq.count(kmer) for kmer in all_kmers])
    return pd.DataFrame(kmer_count_list, columns=all_kmers)


Instruction:
Can you create a Python 3 function that
generates all possible k - mers in a sequence.

Response:
def generate_all_kmers(k):
    """
    Generate all possible k-mers

    Example:
    >>> generate_all_kmers(2)
    ['AA', 'AC', 'AG', 'AT', 'CA', 'CC', 'CG', 'CT', 'GA', 'GC', 'GG', 'GT', 'TA', 'TC', 'TG', 'TT']
    """
    bases = ['A', 'C', 'G', 'T']
    return [''.join(p) for p in itertools.product(bases, repeat=k)]


Instruction:
How would you explain what the following Python 3 function does
def dict_to_numpy_dict(obj_dict):
    return {key: np.asarray(value) if value is not None else None for key, value in obj_dict.items()}

Response: Convert a dictionary of lists into a dictionary of numpy arrays


Instruction:
Can you tell what is the following Python 3 function doing
def rec_dict_to_numpy_dict(obj_dict):
    if type(obj_dict) == dict:
        return {key: rec_dict_to_numpy_dict(value) if value is not None else None for key, value in obj_dict.items()}
    elif obj_dict is None:
        return None
    else:
        return np.asarray(obj_dict)

Response: Recursively converts a dict to a numpy array.


Instruction:
Make a summary of the following Python 3 code
def compare_numpy_dict(a, b, exact=True):
    if type(a) != type(b) and type(a) != np.ndarray and type(b) != np.ndarray:
        return False
    # go through a dictionary
    if type(a) == dict and type(b) == dict:
        if not a.keys() == b.keys():
            return False
        for key in a.keys():
            res = compare_numpy_dict(a[key], b[key], exact)
            if res == False:
                print("false for key = ", key)
                return False
        return True

    # if type(a) == np.ndarray and type(b) == np.ndarray:
    if type(a) == np.ndarray or type(b) == np.ndarray:
        if exact:
            return (a == b).all()
        else:
            return np.testing.assert_almost_equal(a, b)

    if a is None and b is None:
        return True

    raise NotImplementedError

Response: Compare two recursive numpy dictionaries a and b.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_knots(start, end, n_bases=10, spline_order=3):
    x_range = end - start
    start = start - x_range * 0.001
    end = end + x_range * 0.001

    # mgcv annotation
    m = spline_order - 1
    nk = n_bases - m            # number of interior knots

    dknots = (end - start) / (nk - 1)
    knots = np.linspace(start=start - dknots * (m + 1),
                        stop=end + dknots * (m + 1),
                        num=nk + 2 * m + 2)
    return knots.astype(np.float32)

Response: Function to get the knots of a single knot in a single spline.


Instruction:
Given the following Python 3 function, write the documentation
def get_X_spline(x, knots, n_bases=10, spline_order=3, add_intercept=True):
    if len(x.shape) is not 1:
        raise ValueError("x has to be 1 dimentional")

    tck = [knots, np.zeros(n_bases), spline_order]

    X = np.zeros([len(x), n_bases])

    for i in range(n_bases):
        vec = np.zeros(n_bases)
        vec[i] = 1.0
        tck[1] = vec

        X[:, i] = si.splev(x, tck, der=0)

    if add_intercept is True:
        ones = np.ones_like(X[:, :1])
        X = np.hstack([ones, X])

    return X.astype(np.float32)

Response: Returns the X_spline of the knots.


Instruction:
Create a Python 3 function to
get the penalty matrix S Returns np. array of shape n_bases + add_intercept

Response:
def getS(self, add_intercept=False):
        """Get the penalty matrix S

        Returns
            np.array, of shape (n_bases + add_intercept, n_bases + add_intercept)
        """
        S = self.S
        if add_intercept is True:
            # S <- cbind(0, rbind(0, S)) # in R
            zeros = np.zeros_like(S[:1, :])
            S = np.vstack([zeros, S])

            zeros = np.zeros_like(S[:, :1])
            S = np.hstack([zeros, S])
        return S


Instruction:
Make a summary of the following Python 3 code
def get_metadata():
    motifs = _load_motifs()

    motif_names = sorted(list(motifs.keys()))
    df = pd.Series(motif_names).str.split(expand=True)
    df.rename(columns={0: "PWM_id", 1: "info1", 2: "info2"}, inplace=True)

    # compute the consensus
    consensus = pd.Series([PWM(motifs[m]).get_consensus() for m in motif_names])
    df["consensus"] = consensus
    return df

Response: Get pandas. DataFrame with metadata about the PWM s.


Instruction:
Explain what the following Python 3 code does
def get_pwm_list(motif_name_list, pseudocountProb=0.0001):
    l = _load_motifs()
    l = {k.split()[0]: v for k, v in l.items()}
    pwm_list = [PWM(l[m] + pseudocountProb, name=m) for m in motif_name_list]
    return pwm_list

Response: Get a list of ENCODE PWM s.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def auc(y_true, y_pred, round=True):
    y_true, y_pred = _mask_value_nan(y_true, y_pred)

    if round:
        y_true = y_true.round()
    if len(y_true) == 0 or len(np.unique(y_true)) < 2:
        return np.nan
    return skm.roc_auc_score(y_true, y_pred)

Response: Area under the ROC curve


Instruction:
Here you have a function in Python 3, explain what it does
def recall_at_precision(y_true, y_pred, precision):
    y_true, y_pred = _mask_value_nan(y_true, y_pred)
    precision, recall, _ = skm.precision_recall_curve(y_true, y_pred)
    return recall[np.searchsorted(precision - precision, 0)]

Response: Recall at a certain precision threshold


Instruction:
How would you code a function in Python 3 to
true positive rate tp / fn

Response:
def tpr(y_true, y_pred, round=True):
    """True positive rate `tp / (tp + fn)`
    """
    y_true, y_pred = _mask_value_nan(y_true, y_pred)
    if round:
        y_true = np.round(y_true)
        y_pred = np.round(y_pred)
    return skm.recall_score(y_true, y_pred)


Instruction:
How would you implement a function in Python 3 that
trues negative rate tn / fp

Response:
def tnr(y_true, y_pred, round=True):
    """True negative rate `tn / (tn + fp)`
    """
    y_true, y_pred = _mask_value_nan(y_true, y_pred)
    if round:
        y_true = np.round(y_true)
        y_pred = np.round(y_pred)
    c = skm.confusion_matrix(y_true, y_pred)
    return c[0, 0] / c[0].sum()


Instruction:
Given the following Python 3 function, write the documentation
def f1(y_true, y_pred, round=True):
    y_true, y_pred = _mask_value_nan(y_true, y_pred)
    if round:
        y_true = np.round(y_true)
        y_pred = np.round(y_pred)
    return skm.f1_score(y_true, y_pred)

Response: F1 score of a cluster.


Instruction:
Explain what the following Python 3 code does
def cor(y_true, y_pred):
    y_true, y_pred = _mask_nan(y_true, y_pred)
    return np.corrcoef(y_true, y_pred)[0, 1]

Response: Compute Pearson correlation coefficient.


Instruction:
Can you tell what is the following Python 3 function doing
def mad(y_true, y_pred):
    y_true, y_pred = _mask_nan(y_true, y_pred)
    return np.mean(np.abs(y_true - y_pred))

Response: Median absolute deviation of the data.


Instruction:
Given the following Python 3 function, write the documentation
def mse(y_true, y_pred):
    y_true, y_pred = _mask_nan(y_true, y_pred)
    return ((y_true - y_pred) ** 2).mean(axis=None)

Response: Mean squared error of a series of data.


Instruction:
Implement a Python 3 function for
fractioning of variance explained.

Response:
def var_explained(y_true, y_pred):
    """Fraction of variance explained.
    """
    y_true, y_pred = _mask_nan(y_true, y_pred)
    var_resid = np.var(y_true - y_pred)
    var_y_true = np.var(y_true)
    return 1 - var_resid / var_y_true


Instruction:
Make a summary of the following Python 3 code
def contingency_table(y, z):
    y = K.cast(K.round(y), K.floatx())
    z = K.cast(K.round(z), K.floatx())

    def count_matches(y, z):
        return K.sum(K.cast(y, K.floatx()) * K.cast(z, K.floatx()))

    ones = K.ones_like(y)
    zeros = K.zeros_like(y)
    y_ones = K.equal(y, ones)
    y_zeros = K.equal(y, zeros)
    z_ones = K.equal(z, ones)
    z_zeros = K.equal(z, zeros)

    tp = count_matches(y_ones, z_ones)
    tn = count_matches(y_zeros, z_zeros)
    fp = count_matches(y_zeros, z_ones)
    fn = count_matches(y_ones, z_zeros)

    return (tp, tn, fp, fn)

Response: Return the contingency table for a given set of y and z.


Instruction:
Explain what the following Python 3 code does
def tpr(y, z):
    tp, tn, fp, fn = contingency_table(y, z)
    return tp / (tp + fn)

Response: True positive rate y z


Instruction:
Create a Python 3 function for
truing negative rate y / z

Response:
def tnr(y, z):
    """True negative rate `tn / (tn + fp)`
    """
    tp, tn, fp, fn = contingency_table(y, z)
    return tn / (tn + fp)


Instruction:
Make a summary of the following Python 3 code
def accuracy(y, z):
    tp, tn, fp, fn = contingency_table(y, z)
    return (tp + tn) / (tp + tn + fp + fn)

Response: Classification accuracy `(tp + tn) / (tp + tn + fp + fn)`


Instruction:
Can you generate the documentation for the following Python 3 function
def f1(y, z):
    _recall = recall(y, z)
    _prec = precision(y, z)
    return 2 * (_prec * _recall) / (_prec + _recall)

Response: F1 score of a single node in a tree.


Instruction:
Can you tell what is the following Python 3 function doing
def cat_acc(y, z):
    weights = _cat_sample_weights(y)
    _acc = K.cast(K.equal(K.argmax(y, axis=-1),
                          K.argmax(z, axis=-1)),
                  K.floatx())
    _acc = K.sum(_acc * weights) / K.sum(weights)
    return _acc

Response: Classification accuracy for multi - categorical case


Instruction:
How would you explain what the following Python 3 function does
def var_explained(y_true, y_pred):
    var_resid = K.var(y_true - y_pred)
    var_y_true = K.var(y_true)
    return 1 - var_resid / var_y_true

Response: Fraction of variance explained.


Instruction:
Write a Python 3 script for
getting k - fold indices generator

Response:
def split_KFold_idx(train, cv_n_folds=5, stratified=False, random_state=None):
    """Get k-fold indices generator
    """
    test_len(train)
    y = train[1]
    n_rows = y.shape[0]
    if stratified:
        if len(y.shape) > 1:
            if y.shape[1] > 1:
                raise ValueError("Can't use stratified K-fold with multi-column response variable")
            else:
                y = y[:, 0]
            # http://scikit-learn.org/stable/modules/generated/sklearn.model_selection.StratifiedKFold.html#sklearn.model_selection.StratifiedKFold.split
        return model_selection.StratifiedKFold(n_splits=cv_n_folds, shuffle=True, random_state=random_state)\
            .split(X=np.zeros((n_rows, 1)), y=y)
    else:
        return model_selection.KFold(n_splits=cv_n_folds, shuffle=True, random_state=random_state)\
            .split(X=np.zeros((n_rows, 1)))


Instruction:
Explain what the following Python 3 code does
def prepare_data(dt, features, response, sequence, id_column=None, seq_align="end", trim_seq_len=None):
    if type(response) is str:
        response = [response]

    X_feat = np.array(dt[features], dtype="float32")
    y = np.array(dt[response], dtype="float32")
    X_seq = encodeDNA(seq_vec=dt[sequence],
                      maxlen=trim_seq_len,
                      seq_align=seq_align)
    X_seq = np.array(X_seq, dtype="float32")
    id_vec = np.array(dt[id_column])

    return X_feat, X_seq, y, id_vec

Response: Prepare data for Concise. train or ConciseCV. train.


Instruction:
How would you explain what the following Python 3 function does
def _trunc(x, minval=None, maxval=None):
    x = np.copy(x)
    if minval is not None:
        x[x < minval] = minval
    if maxval is not None:
        x[x > maxval] = maxval
    return x

Response: Truncate vector values to have values on range [ minval maxval )


Instruction:
Can you tell what is the following Python 3 function doing
def fit(self, x):
        assert x.ndim > 1
        self.data_min_ = np.min(x, axis=tuple(range(x.ndim - 1)))
        self.data_max_ = np.max(x, axis=tuple(range(x.ndim - 1)))

        if self.share_knots:
            self.data_min_[:] = np.min(self.data_min_)
            self.data_max_[:] = np.max(self.data_max_)

Response: Fit the knot placement to the data.


Instruction:
Make a summary of the following Python 3 code
def transform(self, x, warn=True):
        # 1. split across last dimension
        # 2. re-use ranges
        # 3. Merge
        array_list = [encodeSplines(x[..., i].reshape((-1, 1)),
                                    n_bases=self.n_bases,
                                    spline_order=self.degree,
                                    warn=warn,
                                    start=self.data_min_[i],
                                    end=self.data_max_[i]).reshape(x[..., i].shape + (self.n_bases,))
                      for i in range(x.shape[-1])]
        return np.stack(array_list, axis=-2)

Response: Transform a set of data into a list of values.


Instruction:
Given the following Python 3 function, write the documentation
def InputCodon(seq_length, ignore_stop_codons=True, name=None, **kwargs):
    if ignore_stop_codons:
        vocab = CODONS
    else:
        vocab = CODONS + STOP_CODONS

    assert seq_length % 3 == 0
    return Input((seq_length / 3, len(vocab)), name=name, **kwargs)

Response: Input placeholder for array returned by encodeCodon.


Instruction:
Implement a Python 3 function for
inputting placeholder for array returned by encodeAA.

Response:
def InputAA(seq_length, name=None, **kwargs):
    """Input placeholder for array returned by `encodeAA`

    Wrapper for: `keras.layers.Input((seq_length, 22), name=name, **kwargs)`
    """
    return Input((seq_length, len(AMINO_ACIDS)), name=name, **kwargs)


Instruction:
How would you implement a function in Python 3 that
inputs wrapper for encodeRNAStructure Wrapper for keras. layers. Input

Response:
def InputRNAStructure(seq_length, name=None, **kwargs):
    """Input placeholder for array returned by `encodeRNAStructure`

    Wrapper for: `keras.layers.Input((seq_length, 5), name=name, **kwargs)`
    """
    return Input((seq_length, len(RNAplfold_PROFILES)), name=name, **kwargs)


Instruction:
Here you have a function in Python 3, explain what it does
def InputSplines(seq_length, n_bases=10, name=None, **kwargs):
    return Input((seq_length, n_bases), name=name, **kwargs)

Response: Input placeholder for array returned by encodeSplines.


Instruction:
Given the following Python 3 function, write the documentation
def InputSplines1D(seq_length, n_bases=10, name=None, **kwargs):
    return Input((seq_length, n_bases), name=name, **kwargs)

Response: Input wrapper for encodeSplines.


Instruction:
How would you code a function in Python 3 to
input wrapper around keras. layers. Input class.

Response:
def InputDNAQuantity(seq_length, n_features=1, name=None, **kwargs):
    """Convenience wrapper around `keras.layers.Input`:

    `Input((seq_length, n_features), name=name, **kwargs)`
    """
    return Input((seq_length, n_features), name=name, **kwargs)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def InputDNAQuantitySplines(seq_length, n_bases=10, name="DNASmoothPosition", **kwargs):
    return Input((seq_length, n_bases), name=name, **kwargs)

Response: Convenience wrapper around keras. layers. Input


Instruction:
How would you code a function in Python 3 to
plot weights as a heatmap

Response:
def _plot_weights_heatmap(self, index=None, figsize=None, **kwargs):
        """Plot weights as a heatmap

        index = can be a particular index or a list of indicies
        **kwargs - additional arguments to concise.utils.plot.heatmap
        """
        W = self.get_weights()[0]
        if index is None:
            index = np.arange(W.shape[2])

        fig = heatmap(np.swapaxes(W[:, :, index], 0, 1), plot_name="filter: ",
                      vocab=self.VOCAB, figsize=figsize, **kwargs)
        # plt.show()
        return fig


Instruction:
How would you explain what the following Python 3 function does
def _plot_weights_motif(self, index, plot_type="motif_raw",
                            background_probs=DEFAULT_BASE_BACKGROUND,
                            ncol=1,
                            figsize=None):
        """Index can only be a single int
        """

        w_all = self.get_weights()

        if len(w_all) == 0:
            raise Exception("Layer needs to be initialized first")
        W = w_all[0]
        if index is None:
            index = np.arange(W.shape[2])

        if isinstance(index, int):
            index = [index]
        fig = plt.figure(figsize=figsize)

        if plot_type == "motif_pwm" and plot_type in self.AVAILABLE_PLOTS:
            arr = pssm_array2pwm_array(W, background_probs)
        elif plot_type == "motif_raw" and plot_type in self.AVAILABLE_PLOTS:
            arr = W
        elif plot_type == "motif_pwm_info" and plot_type in self.AVAILABLE_PLOTS:
            quasi_pwm = pssm_array2pwm_array(W, background_probs)
            arr = _pwm2pwm_info(quasi_pwm)
        else:
            raise ValueError("plot_type needs to be from {0}".format(self.AVAILABLE_PLOTS))

        fig = seqlogo_fig(arr, vocab=self.VOCAB_name, figsize=figsize, ncol=ncol, plot_name="filter: ")
        # fig.show()
        return fig

Response: Plot the motifs for a single entry in the sequence.


Instruction:
Write a Python 3 script for
plotting the weights of the current object.

Response:
def plot_weights(self, index=None, plot_type="motif_raw", figsize=None, ncol=1, **kwargs):
        """Plot filters as heatmap or motifs

        index = can be a particular index or a list of indicies
        **kwargs - additional arguments to concise.utils.plot.heatmap
        """

        if "heatmap" in self.AVAILABLE_PLOTS and plot_type == "heatmap":
            return self._plot_weights_heatmap(index=index, figsize=figsize, ncol=ncol, **kwargs)
        elif plot_type[:5] == "motif":
            return self._plot_weights_motif(index=index, plot_type=plot_type, figsize=figsize, ncol=ncol, **kwargs)
        else:
            raise ValueError("plot_type needs to be from {0}".format(self.AVAILABLE_PLOTS))


Instruction:
Can you generate the documentation for the following Python 3 function
def _check_pwm_list(pwm_list):
    for pwm in pwm_list:
        if not isinstance(pwm, PWM):
            raise TypeError("element {0} of pwm_list is not of type PWM".format(pwm))
    return True

Response: Check the input validity of the PWM list.


Instruction:
Can you create a Python 3 function that
adds noise with truncnorm from numpy. random. truncnorm from numpy. random. truncnorm

Response:
def _truncated_normal(mean,
                      stddev,
                      seed=None,
                      normalize=True,
                      alpha=0.01):
    ''' Add noise with truncnorm from numpy.
    Bounded (0.001,0.999)
    '''
    # within range ()
    # provide entry to chose which adding noise way to use
    if seed is not None:
        np.random.seed(seed)
    if stddev == 0:
        X = mean
    else:
        gen_X = truncnorm((alpha - mean) / stddev,
                          ((1 - alpha) - mean) / stddev,
                          loc=mean, scale=stddev)
        X = gen_X.rvs() + mean
        if normalize:
            # Normalize, column sum to 1
            col_sums = X.sum(1)
            X = X / col_sums[:, np.newaxis]
    return X


Instruction:
Create a Python 3 function for
plotting a heatmap from weight matrix w vmin vmax diverge_color = True will plot a heatmap from the first axis.

Response:
def heatmap(w, vmin=None, vmax=None, diverge_color=False,
            ncol=1,
            plot_name=None, vocab=["A", "C", "G", "T"], figsize=(6, 2)):
    """Plot a heatmap from weight matrix w

    vmin, vmax = z axis range
    diverge_color = Should we use diverging colors?
    plot_name = plot_title
    vocab = vocabulary (corresponds to the first axis)
    """
    # Generate y and x values from the dimension lengths
    assert len(vocab) == w.shape[0]
    plt_y = np.arange(w.shape[0] + 1) + 0.5
    plt_x = np.arange(w.shape[1] + 1) - 0.5
    z_min = w.min()
    z_max = w.max()

    if vmin is None:
        vmin = z_min
    if vmax is None:
        vmax = z_max

    if diverge_color:
        color_map = plt.cm.RdBu
    else:
        color_map = plt.cm.Blues

    fig = plt.figure(figsize=figsize)
    # multiple axis
    if len(w.shape) == 3:
        #
        n_plots = w.shape[2]
        nrow = math.ceil(n_plots / ncol)
    else:
        n_plots = 1
        nrow = 1
        ncol = 1

    for i in range(n_plots):
        if len(w.shape) == 3:
            w_cur = w[:, :, i]
        else:
            w_cur = w
        ax = plt.subplot(nrow, ncol, i + 1)
        plt.tight_layout()
        im = ax.pcolormesh(plt_x, plt_y, w_cur, cmap=color_map,
                           vmin=vmin, vmax=vmax, edgecolors="white")
        ax.grid(False)
        ax.set_yticklabels([""] + vocab, minor=False)
        ax.yaxis.set_major_locator(MaxNLocator(integer=True))
        ax.set_xticks(np.arange(w_cur.shape[1] + 1))
        ax.set_xlim(plt_x.min(), plt_x.max())
        ax.set_ylim(plt_y.min(), plt_y.max())

        # nice scale location:
        # http://stackoverflow.com/questions/18195758/set-matplotlib-colorbar-size-to-match-graph
        divider = make_axes_locatable(ax)
        cax = divider.append_axes("right", size="5%", pad=0.05)
        fig.colorbar(im, cax=cax)

        if plot_name is not None:
            if n_plots > 0:
                pln = plot_name + " {0}".format(i)
            else:
                pln = plot_name
            ax.set_title(pln)
        ax.set_aspect('equal')
    return fig


Instruction:
How would you explain what the following Python 3 function does
def standardize_polygons_str(data_str):
    # find all of the polygons in the letter (for instance an A
    # needs to be constructed from 2 polygons)
    path_strs = re.findall("\(\(([^\)]+?)\)\)", data_str.strip())

    # convert the data into a numpy array
    polygons_data = []
    for path_str in path_strs:
        data = np.array([
            tuple(map(float, x.split())) for x in path_str.strip().split(",")])
        polygons_data.append(data)

    # standardize the coordinates
    min_coords = np.vstack(data.min(0) for data in polygons_data).min(0)
    max_coords = np.vstack(data.max(0) for data in polygons_data).max(0)
    for data in polygons_data:
        data[:, ] -= min_coords
        data[:, ] /= (max_coords - min_coords)

    polygons = []
    for data in polygons_data:
        polygons.append(load_wkt(
            "POLYGON((%s))" % ",".join(" ".join(map(str, x)) for x in data)))

    return tuple(polygons)

Response: Given a POLYGON string standardize the coordinates to a 1x1 grid.


Instruction:
Create a Python 3 function to
add a letter to matplotlib axis ax.

Response:
def add_letter_to_axis(ax, let, col, x, y, height):
    """Add 'let' with position x,y and height height to matplotlib axis 'ax'.
    """
    if len(let) == 2:
        colors = [col, "white"]
    elif len(let) == 1:
        colors = [col]
    else:
        raise ValueError("3 or more Polygons are not supported")

    for polygon, color in zip(let, colors):
        new_polygon = affinity.scale(
            polygon, yfact=height, origin=(0, 0, 0))
        new_polygon = affinity.translate(
            new_polygon, xoff=x, yoff=y)
        patch = PolygonPatch(
            new_polygon, edgecolor=color, facecolor=color)
        ax.add_patch(patch)
    return


Instruction:
Can you generate the documentation for the following Python 3 function
def seqlogo(letter_heights, vocab="DNA", ax=None):
    ax = ax or plt.gca()

    assert letter_heights.shape[1] == len(VOCABS[vocab])
    x_range = [1, letter_heights.shape[0]]
    pos_heights = np.copy(letter_heights)
    pos_heights[letter_heights < 0] = 0
    neg_heights = np.copy(letter_heights)
    neg_heights[letter_heights > 0] = 0

    for x_pos, heights in enumerate(letter_heights):
        letters_and_heights = sorted(zip(heights, list(VOCABS[vocab].keys())))
        y_pos_pos = 0.0
        y_neg_pos = 0.0
        for height, letter in letters_and_heights:
            color = VOCABS[vocab][letter]
            polygons = letter_polygons[letter]
            if height > 0:
                add_letter_to_axis(ax, polygons, color, 0.5 + x_pos, y_pos_pos, height)
                y_pos_pos += height
            else:
                add_letter_to_axis(ax, polygons, color, 0.5 + x_pos, y_neg_pos, height)
                y_neg_pos += height

    # if add_hline:
    #     ax.axhline(color="black", linewidth=1)
    ax.set_xlim(x_range[0] - 1, x_range[1] + 1)
    ax.grid(False)
    ax.set_xticks(list(range(*x_range)) + [x_range[-1]])
    ax.set_aspect(aspect='auto', adjustable='box')
    ax.autoscale_view()

Response: Make a sequence logo plot.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_cv_accuracy(res):
    ac_list = [(accuracy["train_acc_final"],
                accuracy["test_acc_final"]
                )
               for accuracy, weights in res]
    ac = np.array(ac_list)

    perf = {
        "mean_train_acc": np.mean(ac[:, 0]),
        "std_train_acc": np.std(ac[:, 0]),
        "mean_test_acc": np.mean(ac[:, 1]),
        "std_test_acc": np.std(ac[:, 1]),

    }
    return perf

Response: Extract the cv accuracy from the model


Instruction:
Implement a function in Python 3 to
convert a one - hot encoded array back to a string

Response:
def one_hot2string(arr, vocab):
    """Convert a one-hot encoded array back to string
    """
    tokens = one_hot2token(arr)
    indexToLetter = _get_index_dict(vocab)

    return [''.join([indexToLetter[x] for x in row]) for row in tokens]


Instruction:
Here you have a function in Python 3, explain what it does
def tokenize(seq, vocab, neutral_vocab=[]):
    # Req: all vocabs have the same length
    if isinstance(neutral_vocab, str):
        neutral_vocab = [neutral_vocab]

    nchar = len(vocab[0])
    for l in vocab + neutral_vocab:
        assert len(l) == nchar
    assert len(seq) % nchar == 0  # since we are using striding

    vocab_dict = _get_vocab_dict(vocab)
    for l in neutral_vocab:
        vocab_dict[l] = -1

    # current performance bottleneck
    return [vocab_dict[seq[(i * nchar):((i + 1) * nchar)]] for i in range(len(seq) // nchar)]

Response:
Convert sequence to integers
sgi


Instruction:
Can you generate the documentation for the following Python 3 function
def token2one_hot(tvec, vocab_size):
    arr = np.zeros((len(tvec), vocab_size))

    tvec_range = np.arange(len(tvec))
    tvec = np.asarray(tvec)
    arr[tvec_range[tvec >= 0], tvec[tvec >= 0]] = 1
    return arr

Response: Convert a token vector into one - hot array.


Instruction:
Write a Python 3 function that can
convert a list of genetic sequences into one - hot - encoded array.

Response:
def encodeSequence(seq_vec, vocab, neutral_vocab, maxlen=None,
                   seq_align="start", pad_value="N", encode_type="one_hot"):
    """Convert a list of genetic sequences into one-hot-encoded array.

    # Arguments
       seq_vec: list of strings (genetic sequences)
       vocab: list of chars: List of "words" to use as the vocabulary. Can be strings of length>0,
            but all need to have the same length. For DNA, this is: ["A", "C", "G", "T"].
       neutral_vocab: list of chars: Values used to pad the sequence or represent unknown-values. For DNA, this is: ["N"].
       maxlen: int or None,
            Should we trim (subset) the resulting sequence. If None don't trim.
            Note that trims wrt the align parameter.
            It should be smaller than the longest sequence.
       seq_align: character; 'end' or 'start'
            To which end should we align sequences?
       encode_type: "one_hot" or "token". "token" represents each vocab element as a positive integer from 1 to len(vocab) + 1.
                  neutral_vocab is represented with 0.

    # Returns
        Array with shape for encode_type:

            - "one_hot": `(len(seq_vec), maxlen, len(vocab))`
            - "token": `(len(seq_vec), maxlen)`

        If `maxlen=None`, it gets the value of the longest sequence length from `seq_vec`.
    """
    if isinstance(neutral_vocab, str):
        neutral_vocab = [neutral_vocab]
    if isinstance(seq_vec, str):
        raise ValueError("seq_vec should be an iterable returning " +
                         "strings not a string itself")
    assert len(vocab[0]) == len(pad_value)
    assert pad_value in neutral_vocab

    assert encode_type in ["one_hot", "token"]

    seq_vec = pad_sequences(seq_vec, maxlen=maxlen,
                            align=seq_align, value=pad_value)

    if encode_type == "one_hot":
        arr_list = [token2one_hot(tokenize(seq, vocab, neutral_vocab), len(vocab))
                    for i, seq in enumerate(seq_vec)]
    elif encode_type == "token":
        arr_list = [1 + np.array(tokenize(seq, vocab, neutral_vocab)) for seq in seq_vec]
        # we add 1 to be compatible with keras: https://keras.io/layers/embeddings/
        # indexes > 0, 0 = padding element

    return np.stack(arr_list)


Instruction:
Here you have a function in Python 3, explain what it does
def encodeDNA(seq_vec, maxlen=None, seq_align="start"):
    return encodeSequence(seq_vec,
                          vocab=DNA,
                          neutral_vocab="N",
                          maxlen=maxlen,
                          seq_align=seq_align,
                          pad_value="N",
                          encode_type="one_hot")

Response: Convert the DNA sequence into 1 - hot - encoding numpy array


Instruction:
Can you implement a function in Python 3 that
converts the RNA sequence into 1 - hot - encoding numpy array as for encodeDNA

Response:
def encodeRNA(seq_vec, maxlen=None, seq_align="start"):
    """Convert the RNA sequence into 1-hot-encoding numpy array as for encodeDNA
    """
    return encodeSequence(seq_vec,
                          vocab=RNA,
                          neutral_vocab="N",
                          maxlen=maxlen,
                          seq_align=seq_align,
                          pad_value="N",
                          encode_type="one_hot")


Instruction:
How would you implement a function in Python 3 that
converts the Codon sequence into 1 - hot - encoding numpy array.

Response:
def encodeCodon(seq_vec, ignore_stop_codons=True, maxlen=None, seq_align="start", encode_type="one_hot"):
    """Convert the Codon sequence into 1-hot-encoding numpy array

    # Arguments
        seq_vec: List of strings/DNA sequences
        ignore_stop_codons: boolean; if True, STOP_CODONS are omitted from one-hot encoding.
        maxlen: Maximum sequence length. See `pad_sequences` for more detail
        seq_align: How to align the sequences of variable lengths. See `pad_sequences` for more detail
        encode_type: can be `"one_hot"` or `token` for token encoding of codons (incremental integer ).

    # Returns
        numpy.ndarray of shape `(len(seq_vec), maxlen / 3, 61 if ignore_stop_codons else 64)`
    """
    if ignore_stop_codons:
        vocab = CODONS
        neutral_vocab = STOP_CODONS + ["NNN"]
    else:
        vocab = CODONS + STOP_CODONS
        neutral_vocab = ["NNN"]

    # replace all U's with A's?
    seq_vec = [str(seq).replace("U", "T") for seq in seq_vec]

    return encodeSequence(seq_vec,
                          vocab=vocab,
                          neutral_vocab=neutral_vocab,
                          maxlen=maxlen,
                          seq_align=seq_align,
                          pad_value="NNN",
                          encode_type=encode_type)


Instruction:
How would you explain what the following Python 3 function does
def encodeAA(seq_vec, maxlen=None, seq_align="start", encode_type="one_hot"):
    return encodeSequence(seq_vec,
                          vocab=AMINO_ACIDS,
                          neutral_vocab="_",
                          maxlen=maxlen,
                          seq_align=seq_align,
                          pad_value="_",
                          encode_type=encode_type)

Response: Convert the Amino - acid sequence into 1 - hot - encoding numpy array.


Instruction:
Can you create a Python 3 function that
returns a list of sequences with common length.

Response:
def pad_sequences(sequence_vec, maxlen=None, align="end", value="N"):
    """Pad and/or trim a list of sequences to have common length. Procedure:

        1. Pad the sequence with N's or any other string or list element (`value`)
        2. Subset the sequence

    # Note
        See also: https://keras.io/preprocessing/sequence/
        Aplicable also for lists of characters

    # Arguments
        sequence_vec: list of chars or lists
            List of sequences that can have various lengths
        value: Neutral element to pad the sequence with. Can be `str` or `list`.
        maxlen: int or None; Final lenght of sequences.
             If None, maxlen is set to the longest sequence length.
        align: character; 'start', 'end' or 'center'
            To which end to align the sequences when triming/padding. See examples bellow.

    # Returns
        List of sequences of the same class as sequence_vec

    # Example

        ```python
            >>> sequence_vec = ['CTTACTCAGA', 'TCTTTA']
            >>> pad_sequences(sequence_vec, 10, align="start", value="N")
            ['CTTACTCAGA', 'TCTTTANNNN']
            >>> pad_sequences(sequence_vec, 10, align="end", value="N")
            ['CTTACTCAGA', 'NNNNTCTTTA']
            >>> pad_sequences(sequence_vec, 4, align="center", value="N")
            ['ACTC', 'CTTT']
        ```
    """

    # neutral element type checking
    assert isinstance(value, list) or isinstance(value, str)
    assert isinstance(value, type(sequence_vec[0]))
    assert not isinstance(sequence_vec, str)
    assert isinstance(sequence_vec[0], list) or isinstance(sequence_vec[0], str)

    max_seq_len = max([len(seq) for seq in sequence_vec])

    if maxlen is None:
        maxlen = max_seq_len
    else:
        maxlen = int(maxlen)

    if max_seq_len < maxlen:
        print("WARNING: Maximum sequence length (%s) is less than maxlen (%s)" % (max_seq_len, maxlen))
        max_seq_len = maxlen

    # check the case when len > 1
    for seq in sequence_vec:
        if not len(seq) % len(value) == 0:
            raise ValueError("All sequences need to be dividable by len(value)")
    if not maxlen % len(value) == 0:
        raise ValueError("maxlen needs to be dividable by len(value)")

    # pad and subset
    def pad(seq, max_seq_len, value="N", align="end"):
        seq_len = len(seq)
        assert max_seq_len >= seq_len
        if align is "end":
            n_left = max_seq_len - seq_len
            n_right = 0
        elif align is "start":
            n_right = max_seq_len - seq_len
            n_left = 0
        elif align is "center":
            n_left = (max_seq_len - seq_len) // 2 + (max_seq_len - seq_len) % 2
            n_right = (max_seq_len - seq_len) // 2
        else:
            raise ValueError("align can be of: end, start or center")

        # normalize for the length
        n_left = n_left // len(value)
        n_right = n_right // len(value)

        return value * n_left + seq + value * n_right

    def trim(seq, maxlen, align="end"):
        seq_len = len(seq)

        assert maxlen <= seq_len
        if align is "end":
            return seq[-maxlen:]
        elif align is "start":
            return seq[0:maxlen]
        elif align is "center":
            dl = seq_len - maxlen
            n_left = dl // 2 + dl % 2
            n_right = seq_len - dl // 2
            return seq[n_left:n_right]
        else:
            raise ValueError("align can be of: end, start or center")

    padded_sequence_vec = [pad(seq, max(max_seq_len, maxlen),
                               value=value, align=align) for seq in sequence_vec]
    padded_sequence_vec = [trim(seq, maxlen, align=align) for seq in padded_sequence_vec]

    return padded_sequence_vec


Instruction:
Here you have a function in Python 3, explain what it does
def extract_landmarks(gtf, landmarks=ALL_LANDMARKS):
    if isinstance(gtf, str):
        _logger.info("Reading gtf file..")
        gtf = read_gtf(gtf)
        _logger.info("Done")

    _logger.info("Running landmark extractors..")
    # landmarks to a dictionary with a function
    assert isinstance(landmarks, (list, tuple, set, dict))
    if isinstance(landmarks, dict):
        landmarks = {k: _get_fun(v) for k, v in landmarks.items()}
    else:
        landmarks = {_to_string(fn_str): _get_fun(fn_str)
                     for fn_str in landmarks}

    r = {k: _validate_pos(v(gtf)) for k, v in landmarks.items()}
    _logger.info("Done!")
    return r

Response: Given a GFF file and a dictionary of landmarks extract the positions of the sequence in that file.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _validate_pos(df):
    assert isinstance(df, pd.DataFrame)
    assert ["seqname", "position", "strand"] == df.columns.tolist()
    assert df.position.dtype == np.dtype("int64")
    assert df.strand.dtype == np.dtype("O")
    assert df.seqname.dtype == np.dtype("O")
    return df

Response: Validates the returned positional object


Instruction:
Can you create a Python 3 function that
defines a huber loss for the current time step.

Response:
def huber_loss(tensor, k=1, scope=None):
    """Define a huber loss  https://en.wikipedia.org/wiki/Huber_loss
      tensor: tensor to regularize.
      k: value of k in the huber loss
      scope: Optional scope for op_scope.

    Huber loss:
    f(x) = if |x| <= k:
              0.5 * x^2
           else:
              k * |x| - 0.5 * k^2

    Returns:
      the L1 loss op.
    """
    # assert k >= 0
    with tf.name_scope(scope, 'L1Loss', [tensor]):
        loss = tf.reduce_mean(tf.select(tf.abs(tensor) < k,
                                        0.5 * tf.square(tensor),
                                        k * tf.abs(tensor) - 0.5 * k ^ 2)
                              )
        return loss


Instruction:
Can you generate the documentation for the following Python 3 function
def get_metadata():
    dt = pd.read_table(ATTRACT_METADTA)
    dt.rename(columns={"Matrix_id": "PWM_id"}, inplace=True)
    # put to firt place
    cols = ['PWM_id'] + [col for col in dt if col != 'PWM_id']

    # rename Matrix_id to PWM_id
    return dt[cols]

Response: Get pandas. DataFrame with metadata about the Attract PWM s.


Instruction:
Write a Python 3 script to
get a list of Attract PWM s.

Response:
def get_pwm_list(pwm_id_list, pseudocountProb=0.0001):
    """Get a list of Attract PWM's.

    # Arguments
        pwm_id_list: List of id's from the `PWM_id` column in `get_metadata()` table
        pseudocountProb: Added pseudocount probabilities to the PWM

    # Returns
        List of `concise.utils.pwm.PWM` instances.
    """
    l = load_motif_db(ATTRACT_PWM)
    l = {k.split()[0]: v for k, v in l.items()}
    pwm_list = [PWM(l[str(m)] + pseudocountProb, name=m) for m in pwm_id_list]
    return pwm_list


Instruction:
Can you generate a brief explanation for the following Python 3 code
def mask_loss(loss, mask_value=MASK_VALUE):
    loss_fn = kloss.deserialize(loss)

    def masked_loss_fn(y_true, y_pred):
        # currently not suppoerd with NA's:
        #  - there is no K.is_nan impolementation in keras.backend
        #  - https://github.com/fchollet/keras/issues/1628
        mask = K.cast(K.not_equal(y_true, mask_value), K.floatx())

        # we divide by the mean to correct for the number of done loss evaluations
        return loss_fn(y_true * mask, y_pred * mask) / K.mean(mask)

    masked_loss_fn.__name__ = loss + "_masked"
    return masked_loss_fn

Response: Generates a new keras loss function that ignores values where y_true == mask_value


Instruction:
Write a Python 3 script to
get a list of HOCOMOCO PWMs.

Response:
def get_pwm_list(pwm_id_list, pseudocountProb=0.0001):
    """Get a list of HOCOMOCO PWM's.

    # Arguments
        pwm_id_list: List of id's from the `PWM_id` column in `get_metadata()` table
        pseudocountProb: Added pseudocount probabilities to the PWM

    # Returns
        List of `concise.utils.pwm.PWM` instances.
    """
    l = load_motif_db(HOCOMOCO_PWM)
    l = {k.split()[0]: v for k, v in l.items()}
    pwm_list = [PWM(_normalize_pwm(l[m]) + pseudocountProb, name=m) for m in pwm_id_list]
    return pwm_list


Instruction:
How would you implement a function in Python 3 that
returns a dictionary of trained weights.

Response:
def get_weights(self):
        """
        Returns:
            dict: Model's trained weights.
        """
        if self.is_trained() is False:
            # print("Model not fitted yet. Use object.fit() to fit the model.")
            return None

        var_res = self._var_res
        weights = self._var_res_to_weights(var_res)
        # save to the side
        weights["final_bias_fit"] = weights["final_bias"]
        weights["feature_weights_fit"] = weights["feature_weights"]

        return weights


Instruction:
How would you explain what the following Python 3 function does
def _var_res_to_weights(self, var_res):
        # transform the weights into our form
        motif_base_weights_raw = var_res["motif_base_weights"][0]
        motif_base_weights = np.swapaxes(motif_base_weights_raw, 0, 2)

        # get weights
        motif_weights = var_res["motif_weights"]
        motif_bias = var_res["motif_bias"]
        final_bias = var_res["final_bias"]
        feature_weights = var_res["feature_weights"]

        # get the GAM prediction:
        spline_pred = None
        spline_weights = None
        if self._param["n_splines"] is not None:
            spline_pred = self._splines["X_spline"].dot(var_res["spline_weights"])

            if self._param["spline_exp"] is True:
                spline_pred = np.exp(spline_pred)
            else:
                spline_pred = (spline_pred + 1)

            spline_pred.reshape([-1])
            spline_weights = var_res["spline_weights"]

        weights = {"motif_base_weights": motif_base_weights,
                   "motif_weights": motif_weights,
                   "motif_bias": motif_bias,
                   "final_bias": final_bias,
                   "feature_weights": feature_weights,
                   "spline_pred": spline_pred,
                   "spline_weights": spline_weights
                   }
        return weights

Response: Convert a var_res to a dict of weights.


Instruction:
How would you explain what the following Python 3 function does
def _get_var_res(self, graph, var, other_var):
        with tf.Session(graph=graph) as sess:
            sess.run(other_var["init"])

            # all_vars = tf.all_variables()
            # print("All variable names")
            # print([var.name for var in all_vars])
            # print("All variable values")
            # print(sess.run(all_vars))
            var_res = self._get_var_res_sess(sess, var)
        return var_res

Response: Get the weights from our graph and the other variable


Instruction:
Write a Python 3 function for
converting a dictionary of numpy arrays to tf. Variables

Response:
def _convert_to_var(self, graph, var_res):
        """
        Create tf.Variables from a list of numpy arrays

        var_res: dictionary of numpy arrays with the key names corresponding to var
        """
        with graph.as_default():
            var = {}
            for key, value in var_res.items():
                if value is not None:
                    var[key] = tf.Variable(value, name="tf_%s" % key)
                else:
                    var[key] = None
        return var


Instruction:
Write a Python 3 function for
training the CONCISE model for the given feature design matrix X_feat X_seq y y_valid is a numpy array of length N_seq.

Response:
def train(self, X_feat, X_seq, y,
              X_feat_valid=None, X_seq_valid=None, y_valid=None,
              n_cores=3):
        """Train the CONCISE model

        :py:attr:`X_feat`, :py:attr:`X_seq`, py:attr:`y` are preferrably returned by the :py:func:`concise.prepare_data` function.

        Args:
            X_feat: Numpy (float) array of shape :code:`(N, D)`. Feature design matrix storing :code:`N` training samples and :code:`D` features
            X_seq:  Numpy (float) array of shape :code:`(N, 1, N_seq, 4)`. It represents 1-hot encoding of the DNA/RNA sequence.(:code:`N`-seqeuences of length :code:`N_seq`)
            y: Numpy (float) array of shape :code:`(N, 1)`. Response variable.
            X_feat_valid: :py:attr:`X_feat` used for model validation.
            X_seq_valid: :py:attr:`X_seq` used for model validation.
            y: :py:attr:`y` used for model validation.
            n_cores (int): Number of CPU cores used for training. If available, GPU is used for training and this argument is ignored.
        """

        if X_feat_valid is None and X_seq_valid is None and y_valid is None:
            X_feat_valid = X_feat
            X_seq_valid = X_seq
            y_valid = y
            print("Using training samples also for validation ")

        # insert one dimension - backcompatiblity
        X_seq = np.expand_dims(X_seq, axis=1)
        X_seq_valid = np.expand_dims(X_seq_valid, axis=1)

        # TODO: implement the re-training feature
        if self.is_trained() is True:
            print("Model already fitted. Re-training feature not implemented yet")
            return

        # input check
        assert X_seq.shape[0] == X_feat.shape[0] == y.shape[0]
        assert y.shape == (X_feat.shape[0], self._num_tasks)

        # extract data specific parameters
        self._param["seq_length"] = X_seq.shape[2]
        self._param["n_add_features"] = X_feat.shape[1]

        # more input check
        if not self._param["seq_length"] == X_seq_valid.shape[2]:
            raise Exception("sequence lengths don't match")

        # setup splines
        if self._param["n_splines"] is not None:
            padd_loss = self._param["motif_length"] - 1  # how much shorter is our sequence, since we don't use padding
            X_spline, S, _ = splines.get_gam_splines(start=0,
                                                     end=self._param["seq_length"] - padd_loss - 1,  # -1 due to zero-indexing
                                                     n_bases=self._param["n_splines"],
                                                     spline_order=3,
                                                     add_intercept=False)
            self._splines = {"X_spline": X_spline,
                             "S": S
                             }

        # setup graph and variables
        self._graph = tf.Graph()
        self._var = self._get_var_initialization(self._graph, X_feat_train=X_feat, y_train=y)
        self._other_var = self._build_graph(self._graph, self._var)
        # TODO: save the intialized parameters
        var_res_init = self._get_var_res(self._graph, self._var, self._other_var)
        self.init_weights = self._var_res_to_weights(var_res=var_res_init)

        # finally train the model
        # - it saves the accuracy

        if self._param["optimizer"] == "adam":
            _train = self._train_adam
        elif self._param["optimizer"] == "lbfgs":
            _train = self._train_lbfgs
        else:
            raise Exception("Optimizer {} not implemented".format(self._param["optimizer"]))

        self._var_res = _train(X_feat, X_seq, y,
                               X_feat_valid, X_seq_valid, y_valid,
                               graph=self._graph, var=self._var, other_var=self._other_var,
                               early_stop_patience=self._param["early_stop_patience"],
                               n_cores=n_cores)

        self._model_fitted = True

        # TODO: maybe:
        # - add y_train_accuracy
        # - y_train

        return True


Instruction:
Can you implement a function in Python 3 that
predicts y or y_pred from inside the tf session.

Response:
def _predict_in_session(self, sess, other_var, X_feat, X_seq, variable="y_pred"):
        """
        Predict y (or any other variable) from inside the tf session. Variable has to be in other_var
        """
        # other_var["tf_X_seq"]: X_seq, tf_y: y,
        feed_dict = {other_var["tf_X_feat"]: X_feat,
                     other_var["tf_X_seq"]: X_seq}

        y_pred = sess.run(other_var[variable], feed_dict=feed_dict)
        return y_pred


Instruction:
Write a Python 3 script to
compute the accuracy from inside the tf session

Response:
def _accuracy_in_session(self, sess, other_var, X_feat, X_seq, y):
        """
        Compute the accuracy from inside the tf session
        """
        y_pred = self._predict_in_session(sess, other_var, X_feat, X_seq)
        return ce.mse(y_pred, y)


Instruction:
How would you explain what the following Python 3 function does
def predict(self, X_feat, X_seq):

        # insert one dimension - backcompatiblity
        X_seq = np.expand_dims(X_seq, axis=1)

        return self._get_other_var(X_feat, X_seq, variable="y_pred")

Response: Predict the response variable y for new input data.


Instruction:
Given the following Python 3 function, write the documentation
def _get_other_var(self, X_feat, X_seq, variable="y_pred"):
        if self.is_trained() is False:
            print("Model not fitted yet. Use object.fit() to fit the model.")
            return

        # input check:
        assert X_seq.shape[0] == X_feat.shape[0]

        # TODO - check this
        # sequence can be wider or thinner?
        # assert self._param["seq_length"] == X_seq.shape[2]
        assert self._param["n_add_features"] == X_feat.shape[1]

        # setup graph and variables
        graph = tf.Graph()
        var = self._convert_to_var(graph, self._var_res)
        other_var = self._build_graph(graph, var)
        with tf.Session(graph=graph) as sess:
            sess.run(other_var["init"])
            # predict
            y = self._predict_in_session(sess, other_var, X_feat, X_seq, variable)

        return y

Response: Get the value of a variable from the other_vars


Instruction:
How would you implement a function in Python 3 that
returns the current object as a dictionary.

Response:
def to_dict(self):
        """
        Returns:
            dict: Concise represented as a dictionary.
        """
        final_res = {
            "param": self._param,
            "unused_param": self.unused_param,
            "execution_time": self._exec_time,
            "output": {"accuracy": self.get_accuracy(),
                       "weights": self.get_weights(),
                       "splines": self._splines
                       }
        }
        return final_res


Instruction:
Given the following Python 3 function, write the documentation
def _set_var_res(self, weights):
        if weights is None:
            return

        # layer 1
        motif_base_weights_raw = np.swapaxes(weights["motif_base_weights"], 2, 0)
        motif_base_weights = motif_base_weights_raw[np.newaxis]
        motif_bias = weights["motif_bias"]

        feature_weights = weights["feature_weights"]
        spline_weights = weights["spline_weights"]

        # filter
        motif_weights = weights["motif_weights"]
        final_bias = weights["final_bias"]

        var_res = {
            "motif_base_weights": motif_base_weights,
            "motif_bias": motif_bias,
            "spline_weights": spline_weights,
            "feature_weights": feature_weights,
            "motif_weights": motif_weights,
            "final_bias": final_bias
        }

        # cast everything to float32
        var_res = {key: value.astype(np.float32) if value is not None else None for key, value in var_res.items()}

        self._var_res = var_res

Response: Transform the weights to var_res


Instruction:
Given the following Python 3 function, write the documentation
def from_dict(cls, obj_dict):

        # convert the output into a proper form
        obj_dict['output'] = helper.rec_dict_to_numpy_dict(obj_dict["output"])

        helper.dict_to_numpy_dict(obj_dict['output'])
        if "trained_global_model" in obj_dict.keys():
            raise Exception("Found trained_global_model feature in dictionary. Use ConciseCV.load to load this file.")
        dc = Concise(**obj_dict["param"])

        # touch the hidden arguments
        dc._param = obj_dict["param"]
        if obj_dict["output"]["weights"] is None:
            dc._model_fitted = False
        else:
            dc._model_fitted = True
            dc._exec_time = obj_dict["execution_time"]

        dc.unused_param = obj_dict["unused_param"]
        dc._accuracy = obj_dict["output"]["accuracy"]
        dc._splines = obj_dict["output"]["splines"]

        weights = obj_dict["output"]["weights"]

        if weights is not None:
            # fix the dimensionality of X_feat in case it was 0 dimensional
            if weights["feature_weights"].shape == (0,):
                weights["feature_weights"].shape = (0, obj_dict["param"]["num_tasks"])
            dc._set_var_res(weights)

        return dc

Response: Load the object from a dictionary.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def load(cls, file_path):

        # convert back to numpy
        data = helper.read_json(file_path)
        return Concise.from_dict(data)

Response: Load the object from a JSON file.


Instruction:
Create a Python 3 function for
getting the used CV folds

Response:
def _get_folds(n_rows, n_folds, use_stored):
        """
        Get the used CV folds
        """
        # n_folds = self._n_folds
        # use_stored = self._use_stored_folds
        # n_rows = self._n_rows

        if use_stored is not None:
            # path = '~/concise/data-offline/lw-pombe/cv_folds_5.json'
            with open(os.path.expanduser(use_stored)) as json_file:
                json_data = json.load(json_file)

            # check if we have the same number of rows and folds:
            if json_data['N_rows'] != n_rows:
                raise Exception('N_rows from folds doesnt match the number of rows of X_seq, X_feat, y')

            if json_data['N_folds'] != n_folds:
                raise Exception('n_folds dont match', json_data['N_folds'], n_folds)

            kf = [(np.array(train), np.array(test)) for (train, test) in json_data['folds']]
        else:
            kf = KFold(n_splits=n_folds).split(np.zeros((n_rows, 1)))

        # store in a list
        i = 1
        folds = []
        for train, test in kf:
            fold = "fold_" + str(i)
            folds.append((fold, train, test))
            i = i + 1
        return folds


Instruction:
Implement a Python 3 function for
training the Concise model in cross - validation.

Response:
def train(self, X_feat, X_seq, y, id_vec=None, n_folds=10, use_stored_folds=None, n_cores=1,
              train_global_model=False):
        """Train the Concise model in cross-validation.

        Args:
            X_feat: See :py:func:`concise.Concise.train`
            X_seq: See :py:func:`concise.Concise.train`
            y: See :py:func:`concise.Concise.train`
            id_vec: List of character id's used to differentiate the trainig samples. Returned by :py:func:`concise.prepare_data`.
            n_folds (int): Number of CV-folds to use.
            use_stored_folds (chr or None): File path to a .json file containing the fold information (as returned by :py:func:`concise.ConciseCV.get_folds`). If None, the folds are generated.
            n_cores (int): Number of CPU cores used for training. If available, GPU is used for training and this argument is ignored.
            train_global_model (bool): In addition to training the model in cross-validation, should the global model be fitted (using all the samples from :code:`(X_feat, X_seq, y)`).
        """
        # TODO: input check - dimensions
        self._use_stored_folds = use_stored_folds
        self._n_folds = n_folds
        self._n_rows = X_feat.shape[0]

    # TODO: - fix the get_cv_accuracy
    # save:
    # - each model
    # - each model's performance
    # - each model's predictions
    # - globally:
    #    - mean perfomance
    #    - sd performance
    #    - predictions

        self._kf = self._get_folds(self._n_rows, self._n_folds, self._use_stored_folds)

        cv_obj = {}

        if id_vec is None:
            id_vec = np.arange(1, self._n_rows + 1)

        best_val_acc_epoch_l = []
        for fold, train, test in self._kf:
            X_feat_train = X_feat[train]
            X_seq_train = X_seq[train]
            y_train = y[train]

            X_feat_test = X_feat[test]
            X_seq_test = X_seq[test]
            y_test = y[test]
            id_vec_test = id_vec[test]
            print(fold, "/", n_folds)

            # copy the object
            dc = copy.deepcopy(self._concise_model)
            dc.train(X_feat_train, X_seq_train, y_train,
                     X_feat_test, X_seq_test, y_test,
                     n_cores=n_cores
                     )

            dc._test(X_feat_test, X_seq_test, y_test, id_vec_test)
            cv_obj[fold] = dc
            best_val_acc_epoch_l.append(dc.get_accuracy()["best_val_acc_epoch"])
        self._cv_model = cv_obj

        # additionaly train the global model
        if train_global_model:
            dc = copy.deepcopy(self._concise_model)

            # overwrite n_epochs with the best average number of best epochs
            dc._param["n_epochs"] = int(np.array(best_val_acc_epoch_l).mean())
            print("tranining global model with n_epochs = " + str(dc._param["n_epochs"]))

            dc.train(X_feat, X_seq, y,
                     n_cores=n_cores
                     )
            dc._test(X_feat, X_seq, y, id_vec)
            self._concise_global_model = dc


Instruction:
How would you explain what the following Python 3 function does
def get_CV_prediction(self):
        # TODO: get it from the test_prediction ...
        # test_id, prediction
        # sort by test_id
        predict_vec = np.zeros((self._n_rows, self._concise_model._num_tasks))
        for fold, train, test in self._kf:
            acc = self._cv_model[fold].get_accuracy()
            predict_vec[test, :] = acc["y_test_prediction"]
        return predict_vec

Response: Returns the predicted test_prediction for all hold - out folds.


Instruction:
Can you implement a function in Python 3 that
returns the prediction accuracy in CV.

Response:
def get_CV_accuracy(self):
        """
        Returns:
            float: Prediction accuracy in CV.
        """
        accuracy = {}
        for fold, train, test in self._kf:
            acc = self._cv_model[fold].get_accuracy()
            accuracy[fold] = acc["test_acc_final"]
        return accuracy


Instruction:
Can you tell what is the following Python 3 function doing
def to_dict(self):
        param = {
            "n_folds": self._n_folds,
            "n_rows": self._n_rows,
            "use_stored_folds": self._use_stored_folds
        }

        if self._concise_global_model is None:
            trained_global_model = None
        else:
            trained_global_model = self._concise_global_model.to_dict()

        obj_dict = {"param": param,
                    "folds": self._kf,
                    "init_model": self._concise_model.to_dict(),
                    "trained_global_model": trained_global_model,
                    "output": {fold: model.to_dict() for fold, model in self.get_CV_models().items()}
                    }
        return obj_dict

Response: Returns a dictionary representation of the ConciseCV.


Instruction:
Can you tell what is the following Python 3 function doing
def from_dict(cls, obj_dict):
        default_model = Concise()
        cvdc = ConciseCV(default_model)
        cvdc._from_dict(obj_dict)
        return cvdc

Response: Load the object from a dictionary.


Instruction:
Here you have a function in Python 3, explain what it does
def _from_dict(self, obj_dict):
        self._n_folds = obj_dict["param"]["n_folds"]
        self._n_rows = obj_dict["param"]["n_rows"]
        self._use_stored_folds = obj_dict["param"]["use_stored_folds"]

        self._concise_model = Concise.from_dict(obj_dict["init_model"])

        if obj_dict["trained_global_model"] is None:
            self._concise_global_model = None
        else:
            self._concise_global_model = Concise.from_dict(obj_dict["trained_global_model"])

        self._kf = [(fold, np.asarray(train), np.asarray(test)) for fold, train, test in obj_dict["folds"]]
        self._cv_model = {fold: Concise.from_dict(model_dict) for fold, model_dict in obj_dict["output"].items()}

Response: Initialize a model from a dictionary.


Instruction:
Create a Python 3 function for
loading the object from a JSON file.

Response:
def load(cls, file_path):
        """
        Load the object from a JSON file (saved with :py:func:`ConciseCV.save`)

        Returns:
            ConciseCV: Loaded ConciseCV object.
        """
        data = helper.read_json(file_path)
        return ConciseCV.from_dict(data)


Instruction:
Can you create a Python 3 function that
converts pwm array to pssm array

Response:
def pwm_array2pssm_array(arr, background_probs=DEFAULT_BASE_BACKGROUND):
    """Convert pwm array to pssm array
    """
    b = background_probs2array(background_probs)
    b = b.reshape([1, 4, 1])
    return np.log(arr / b).astype(arr.dtype)


Instruction:
Write a Python 3 script to
convert pssm array to pwm array

Response:
def pssm_array2pwm_array(arr, background_probs=DEFAULT_BASE_BACKGROUND):
    """Convert pssm array to pwm array
    """
    b = background_probs2array(background_probs)
    b = b.reshape([1, 4, 1])
    return (np.exp(arr) * b).astype(arr.dtype)


Instruction:
Write a Python 3 function for
reading the motif file in the following format and return a dictionary of numpy arrays.

Response:
def load_motif_db(filename, skipn_matrix=0):
    """Read the motif file in the following format

    ```
    >motif_name
    <skip n>0.1<delim>0.2<delim>0.5<delim>0.6
    ...
    >motif_name2
    ....
    ```

    Delim can be anything supported by np.loadtxt

    # Arguments
        filename: str, file path
        skipn_matrix: integer, number of characters to skip when reading
    the numeric matrix (for Encode = 2)

    # Returns
        Dictionary of numpy arrays

    """

    # read-lines
    if filename.endswith(".gz"):
        f = gzip.open(filename, 'rt', encoding='utf-8')
    else:
        f = open(filename, 'r')
    lines = f.readlines()
    f.close()

    motifs_dict = {}
    motif_lines = ""
    motif_name = None

    def lines2matrix(lines):
        return np.loadtxt(StringIO(lines))

    for line in lines:
        if line.startswith(">"):
            if motif_lines:
                # lines -> matrix
                motifs_dict[motif_name] = lines2matrix(motif_lines)
            motif_name = line[1:].strip()
            motif_lines = ""
        else:
            motif_lines += line[skipn_matrix:]

    if motif_lines and motif_name is not None:
        motifs_dict[motif_name] = lines2matrix(motif_lines)

    return motifs_dict


Instruction:
Can you create a Python 3 function that
returns an iterator over the fasta file Given a fasta file yield tuples of header sequence MimeType

Response:
def iter_fasta(file_path):
    """Returns an iterator over the fasta file

    Given a fasta file. yield tuples of header, sequence

    Code modified from Brent Pedersen's:
    "Correct Way To Parse A Fasta File In Python"

    # Example

        ```python
            fasta = fasta_iter("hg19.fa")
            for header, seq in fasta:
               print(header)
        ```
    """
    fh = open(file_path)

    # ditch the boolean (x[0]) and just keep the header or sequence since
    # we know they alternate.
    faiter = (x[1] for x in groupby(fh, lambda line: line[0] == ">"))

    for header in faiter:
        # drop the ">"
        headerStr = header.__next__()[1:].strip()
        # join all sequence lines to one.
        seq = "".join(s.strip() for s in faiter.__next__())
        yield (headerStr, seq)


Instruction:
How would you explain what the following Python 3 function does
def write_fasta(file_path, seq_list, name_list=None):
    if name_list is None:
        name_list = [str(i) for i in range(len(seq_list))]

    # needs to be dict or seq
    with open(file_path, "w") as f:
        for i in range(len(seq_list)):
            f.write(">" + name_list[i] + "\n" + seq_list[i] + "\n")

Response: Writes a fasta file containing the sequence list.


Instruction:
Can you implement a function in Python 3 that
runs RNA - PLL fold on the input FASTA file.

Response:
def run_RNAplfold(input_fasta, tmpdir, W=240, L=160, U=1):
    """
    Arguments:
       W, Int: span - window length
       L, Int, maxiumm span
       U, Int, size of unpaired region
    """

    profiles = RNAplfold_PROFILES_EXECUTE
    for i, P in enumerate(profiles):
        print("running {P}_RNAplfold... ({i}/{N})".format(P=P, i=i + 1, N=len(profiles)))

        command = "{bin}/{P}_RNAplfold".format(bin=RNAplfold_BIN_DIR, P=P)
        file_out = "{tmp}/{P}_profile.fa".format(tmp=tmpdir, P=P)
        args = " -W {W} -L {L} -u {U} < {fa} > {file_out}".format(W=W, L=L, U=U, fa=input_fasta, file_out=file_out)

        os.system(command + args)

        # check if the file is empty
        if os.path.getsize(file_out) == 0:
            raise Exception("command wrote an empty file: {0}".format(file_out))
    print("done!")


Instruction:
Write a Python 3 script for
reading RNAplfold files and return a numpy array of the 2ndary structure.

Response:
def read_RNAplfold(tmpdir, maxlen=None, seq_align="start", pad_with="E"):
    """
    pad_with = with which 2ndary structure should we pad the sequence?
    """
    assert pad_with in {"P", "H", "I", "M", "E"}

    def read_profile(tmpdir, P):
        return [values.strip().split("\t")
                for seq_name, values in iter_fasta("{tmp}/{P}_profile.fa".format(tmp=tmpdir, P=P))]

    def nelem(P, pad_width):
        """get the right neutral element
        """
        return 1 if P is pad_with else 0

    arr_hime = np.array([pad_sequences(read_profile(tmpdir, P),
                                       value=[nelem(P, pad_with)],
                                       align=seq_align,
                                       maxlen=maxlen)
                         for P in RNAplfold_PROFILES_EXECUTE], dtype="float32")

    # add the pairness column
    arr_p = 1 - arr_hime.sum(axis=0)[np.newaxis]
    arr = np.concatenate((arr_p, arr_hime))

    # reshape to: seq, seq_length, num_channels
    arr = np.moveaxis(arr, 0, 2)
    return arr


Instruction:
Write a Python 3 script for
computing the RNA secondary structure of a sequence vector.

Response:
def encodeRNAStructure(seq_vec, maxlen=None, seq_align="start",
                       W=240, L=160, U=1,
                       tmpdir="/tmp/RNAplfold/"):
    """Compute RNA secondary structure with RNAplfold implemented in
    Kazan et al 2010, [doi](https://doi.org/10.1371/journal.pcbi.1000832).

    # Note
        Secondary structure is represented as the probability
        to be in the following states:
        - `["Pairedness", "Hairpin loop", "Internal loop", "Multi loop", "External region"]`
        See Kazan et al 2010, [doi](https://doi.org/10.1371/journal.pcbi.1000832)
        for more information.

    # Arguments
         seq_vec: list of DNA/RNA sequences
         maxlen: Maximum sequence length. See `concise.preprocessing.pad_sequences` for more detail
         seq_align: How to align the sequences of variable lengths. See `concise.preprocessing.pad_sequences` for more detail
         W: Int; span - window length
         L: Int; maxiumm span
         U: Int; size of unpaired region
         tmpdir: Where to store the intermediary files of RNAplfold.

    # Note
        Recommended parameters:

        - for human, mouse use W, L, u : 240, 160, 1
        - for fly, yeast   use W, L, u :  80,  40, 1

    # Returns
        np.ndarray of shape `(len(seq_vec), maxlen, 5)`
    """
    # extend the tmpdir with uuid string to allow for parallel execution
    tmpdir = tmpdir + "/" + str(uuid4()) + "/"

    if not isinstance(seq_vec, list):
        seq_vec = seq_vec.tolist()
    if not os.path.exists(tmpdir):
        os.makedirs(tmpdir)

    fasta_path = tmpdir + "/input.fasta"
    write_fasta(fasta_path, seq_vec)
    run_RNAplfold(fasta_path, tmpdir, W=W, L=L, U=U)
    # 1. split the fasta into pieces
    # 2. run_RNAplfold for each of them
    # 3. Read the results
    return read_RNAplfold(tmpdir, maxlen, seq_align=seq_align, pad_with="E")


Instruction:
Make a summary of the following Python 3 code
def ism(model, ref, ref_rc, alt, alt_rc, mutation_positions, out_annotation_all_outputs,
        output_filter_mask=None, out_annotation=None, diff_type="log_odds", rc_handling="maximum"):
    """In-silico mutagenesis

    Using ISM in with diff_type 'log_odds' and rc_handling 'maximum' will produce predictions as used
    in [DeepSEA](http://www.nature.com/nmeth/journal/v12/n10/full/nmeth.3547.html). ISM offers two ways to
    calculate the difference between the outputs created by reference and alternative sequence and two
    different methods to select whether to use the output generated from the forward or from the
    reverse-complement sequences. To calculate "e-values" as mentioned in DeepSEA the same ISM prediction
    has to be performed on a randomised set of 1 million 1000genomes, MAF-matched variants to get a
    background of predicted effects of random SNPs.

    # Arguments
        model: Keras model
        ref: Input sequence with the reference genotype in the mutation position
        ref_rc: Reverse complement of the 'ref' argument
        alt: Input sequence with the alternative genotype in the mutation position
        alt_rc: Reverse complement of the 'alt' argument
        mutation_positions: Position on which the mutation was placed in the forward sequences
        out_annotation_all_outputs: Output labels of the model.
        output_filter_mask: Mask of boolean values indicating which model outputs should be used.
            Use this or 'out_annotation'
        out_annotation: List of outputs labels for which of the outputs (in case of a multi-task model) the
            predictions should be calculated.
        diff_type: "log_odds" or "diff". When set to 'log_odds' calculate scores based on log_odds, which assumes
            the model output is a probability. When set to 'diff' the model output for 'ref' is subtracted
            from 'alt'. Using 'log_odds' with outputs that are not in the range [0,1] nan will be returned.
        rc_handling: "average" or "maximum". Either average over the predictions derived from forward and
            reverse-complement predictions ('average') or pick the prediction with the bigger absolute
            value ('maximum').

    # Returns
        Dictionary with the key `ism` which contains a pandas DataFrame containing the calculated values
            for each (selected) model output and input sequence
    """

    seqs = {"ref": ref, "ref_rc": ref_rc, "alt": alt, "alt_rc": alt_rc}
    assert diff_type in ["log_odds", "diff"]
    assert rc_handling in ["average", "maximum"]
    assert np.all([np.array(get_seq_len(ref)) == np.array(get_seq_len(seqs[k])) for k in seqs.keys() if k != "ref"])
    assert get_seq_len(ref)[0] == mutation_positions.shape[0]
    assert len(mutation_positions.shape) == 1

    # determine which outputs should be selected
    if output_filter_mask is None:
        if out_annotation is None:
            output_filter_mask = np.arange(out_annotation_all_outputs.shape[0])
        else:
            output_filter_mask = np.where(np.in1d(out_annotation_all_outputs, out_annotation))[0]

    # make sure the labels are assigned correctly
    out_annotation = out_annotation_all_outputs[output_filter_mask]

    preds = {}
    for k in seqs:
        # preds[k] = model.predict(seqs[k])
        preds[k] = np.array(model.predict(seqs[k])[..., output_filter_mask])

    if diff_type == "log_odds":
        if np.any([(preds[k].min() < 0 or preds[k].max() > 1) for k in preds]):
            warnings.warn("Using log_odds on model outputs that are not bound [0,1]")
        diffs = np.log(preds["alt"] / (1 - preds["alt"])) - np.log(preds["ref"] / (1 - preds["ref"]))
        diffs_rc = np.log(preds["alt_rc"] / (1 - preds["alt_rc"])) - np.log(preds["ref_rc"] / (1 - preds["ref_rc"]))
    elif diff_type == "diff":
        diffs = preds["alt"] - preds["ref"]
        diffs_rc = preds["alt_rc"] - preds["ref_rc"]

    if rc_handling == "average":
        diffs = np.mean([diffs, diffs_rc], axis=0)
    elif rc_handling == "maximum":
        replace_filt = np.abs(diffs) < np.abs(diffs_rc)
        diffs[replace_filt] = diffs_rc[replace_filt]

    diffs = pd.DataFrame(diffs, columns=out_annotation)

    return {"ism": diffs}

Response: This function is used to calculate the difference between the outputs of the model output for the given reference and alternative sequences. The model output for the given reference and alternative sequences is used to calculate the predictions of the model output for the given model output.


Instruction:
Implement a function in Python 3 to
fit and evaluate a keras model for training

Response:
def _train_and_eval_single(train, valid, model,
                           batch_size=32, epochs=300, use_weight=False,
                           callbacks=[], eval_best=False, add_eval_metrics={}):
    """Fit and evaluate a keras model

    eval_best: if True, load the checkpointed model for evaluation
    """
    def _format_keras_history(history):
        """nicely format keras history
        """
        return {"params": history.params,
                "loss": merge_dicts({"epoch": history.epoch}, history.history),
                }
    if use_weight:
        sample_weight = train[2]
    else:
        sample_weight = None
    # train the model
    logger.info("Fit...")
    history = History()
    model.fit(train[0], train[1],
              batch_size=batch_size,
              validation_data=valid[:2],
              epochs=epochs,
              sample_weight=sample_weight,
              verbose=2,
              callbacks=[history] + callbacks)

    # get history
    hist = _format_keras_history(history)
    # load and eval the best model
    if eval_best:
        mcp = [x for x in callbacks if isinstance(x, ModelCheckpoint)]
        assert len(mcp) == 1
        model = load_model(mcp[0].filepath)

    return eval_model(model, valid, add_eval_metrics), hist


Instruction:
Can you tell what is the following Python 3 function doing
def eval_model(model, test, add_eval_metrics={}):
    # evaluate the model
    logger.info("Evaluate...")
    # - model_metrics
    model_metrics_values = model.evaluate(test[0], test[1], verbose=0,
                                          batch_size=test[1].shape[0])
    # evaluation is done in a single pass to have more precise metics
    model_metrics = dict(zip(_listify(model.metrics_names),
                             _listify(model_metrics_values)))
    # - eval_metrics
    y_true = test[1]
    y_pred = model.predict(test[0], verbose=0)
    eval_metrics = {k: v(y_true, y_pred) for k, v in add_eval_metrics.items()}

    # handle the case where the two metrics names intersect
    # - omit duplicates from eval_metrics
    intersected_keys = set(model_metrics).intersection(set(eval_metrics))
    if len(intersected_keys) > 0:
        logger.warning("Some metric names intersect: {0}. Ignoring the add_eval_metrics ones".
                       format(intersected_keys))
        eval_metrics = _delete_keys(eval_metrics, intersected_keys)

    return merge_dicts(model_metrics, eval_metrics)

Response: Evaluate the Keras model on the test - set.


Instruction:
Can you implement a function in Python 3 that
feeds model_fn with train_data and param

Response:
def get_model(model_fn, train_data, param):
    """Feed model_fn with train_data and param
    """
    model_param = merge_dicts({"train_data": train_data}, param["model"], param.get("shared", {}))
    return model_fn(**model_param)


Instruction:
Can you generate the documentation for the following Python 3 function
def _delete_keys(dct, keys):
    c = deepcopy(dct)
    assert isinstance(keys, list)
    for k in keys:
        c.pop(k)
    return c

Response: Returns a copy of dct with keys removed


Instruction:
Make a summary of the following Python 3 code
def _mean_dict(dict_list):
    return {k: np.array([d[k] for d in dict_list]).mean()
            for k in dict_list[0].keys()}

Response: Compute the mean value across a list of dictionaries


Instruction:
Write a Python 3 function that can
retrieve trial by tid

Response:
def get_trial(self, tid):
        """Retrieve trial by tid
        """
        lid = np.where(np.array(self.tids) == tid)[0][0]
        return self.trials[lid]


Instruction:
Can you write a function in Python 3 where it
extends the original object to inject checking for stalled jobs and killing them if they are still running for too long

Response:
def count_by_state_unsynced(self, arg):
        """Extends the original object in order to inject checking
        for stalled jobs and killing them if they are running for too long
        """
        if self.kill_timeout is not None:
            self.delete_running(self.kill_timeout)
        return super(CMongoTrials, self).count_by_state_unsynced(arg)


Instruction:
Write a Python 3 function that can
delete jobs stalled in the running state for too long

Response:
def delete_running(self, timeout_last_refresh=0, dry_run=False):
        """Delete jobs stalled in the running state for too long

        timeout_last_refresh, int: number of seconds
        """
        running_all = self.handle.jobs_running()
        running_timeout = [job for job in running_all
                           if coarse_utcnow() > job["refresh_time"] +
                           timedelta(seconds=timeout_last_refresh)]
        if len(running_timeout) == 0:
            # Nothing to stop
            self.refresh_tids(None)
            return None

        if dry_run:
            logger.warning("Dry run. Not removing anything.")

        logger.info("Removing {0}/{1} running jobs. # all jobs: {2} ".
                    format(len(running_timeout), len(running_all), len(self)))

        now = coarse_utcnow()
        logger.info("Current utc time: {0}".format(now))
        logger.info("Time horizont: {0}".format(now - timedelta(seconds=timeout_last_refresh)))
        for job in running_timeout:
            logger.info("Removing job: ")
            pjob = job.to_dict()
            del pjob["misc"]  # ignore misc when printing
            logger.info(pprint.pformat(pjob))
            if not dry_run:
                self.handle.delete(job)
                logger.info("Job deleted")
        self.refresh_tids(None)


Instruction:
Implement a function in Python 3 to
get train history as pd. DataFrame

Response:
def train_history(self, tid=None):
        """Get train history as pd.DataFrame
        """

        def result2history(result):
            if isinstance(result["history"], list):
                return pd.concat([pd.DataFrame(hist["loss"]).assign(fold=i)
                                  for i, hist in enumerate(result["history"])])
            else:
                return pd.DataFrame(result["history"]["loss"])

        # use all
        if tid is None:
            tid = self.valid_tid()

        res = [result2history(t["result"]).assign(tid=t["tid"]) for t in self.trials
               if t["tid"] in _listify(tid)]
        df = pd.concat(res)

        # reorder columns
        fold_name = ["fold"] if "fold" in df else []
        df = _put_first(df, ["tid"] + fold_name + ["epoch"])
        return df


Instruction:
Create a Python 3 function to
return a pd. DataFrame view of the whole experiment

Response:
def as_df(self, ignore_vals=["history"], separator=".", verbose=True):
        """Return a pd.DataFrame view of the whole experiment
        """

        def add_eval(res):
            if "eval" not in res:
                if isinstance(res["history"], list):
                    # take the average across all folds
                    eval_names = list(res["history"][0]["loss"].keys())
                    eval_metrics = np.array([[v[-1] for k, v in hist["loss"].items()]
                                             for hist in res["history"]]).mean(axis=0).tolist()
                    res["eval"] = {eval_names[i]: eval_metrics[i] for i in range(len(eval_metrics))}
                else:
                    res["eval"] = {k: v[-1] for k, v in res["history"]["loss"].items()}
            return res

        def add_n_epoch(df):
            df_epoch = self.train_history().groupby("tid")["epoch"].max().reset_index()
            df_epoch.rename(columns={"epoch": "n_epoch"}, inplace=True)
            return pd.merge(df, df_epoch, on="tid", how="left")

        results = self.get_ok_results(verbose=verbose)
        rp = [_flatten_dict(_delete_keys(add_eval(x), ignore_vals), separator) for x in results]
        df = pd.DataFrame.from_records(rp)

        df = add_n_epoch(df)

        first = ["tid", "loss", "status"]
        return _put_first(df, first)


Instruction:
Can you implement a function in Python 3 that
lists all trades in a market

Response:
def trades(ctx, market, limit, start, stop):
    """ List trades in a market
    """
    market = Market(market, bitshares_instance=ctx.bitshares)
    t = [["time", "quote", "base", "price"]]
    for trade in market.trades(limit, start=start, stop=stop):
        t.append(
            [
                str(trade["time"]),
                str(trade["quote"]),
                str(trade["base"]),
                "{:f} {}/{}".format(
                    trade["price"],
                    trade["base"]["asset"]["symbol"],
                    trade["quote"]["asset"]["symbol"],
                ),
            ]
        )
    print_table(t)


Instruction:
Can you tell what is the following Python 3 function doing
def ticker(ctx, market):
    market = Market(market, bitshares_instance=ctx.bitshares)
    ticker = market.ticker()
    t = [["key", "value"]]
    for key in ticker:
        t.append([key, str(ticker[key])])
    print_table(t)

Response: Show ticker of a market


Instruction:
Can you tell what is the following Python 3 function doing
def cancel(ctx, orders, account):
    print_tx(ctx.bitshares.cancel(orders, account=account))

Response: Cancel one or multiple orders


Instruction:
Given the following Python 3 function, write the documentation
def orderbook(ctx, market):
    market = Market(market, bitshares_instance=ctx.bitshares)
    orderbook = market.orderbook()
    ta = {}
    ta["bids"] = [["quote", "sum quote", "base", "sum base", "price"]]
    cumsumquote = Amount(0, market["quote"])
    cumsumbase = Amount(0, market["base"])
    for order in orderbook["bids"]:
        cumsumbase += order["base"]
        cumsumquote += order["quote"]
        ta["bids"].append(
            [
                str(order["quote"]),
                str(cumsumquote),
                str(order["base"]),
                str(cumsumbase),
                "{:f} {}/{}".format(
                    order["price"],
                    order["base"]["asset"]["symbol"],
                    order["quote"]["asset"]["symbol"],
                ),
            ]
        )

    ta["asks"] = [["price", "base", "sum base", "quote", "sum quote"]]
    cumsumquote = Amount(0, market["quote"])
    cumsumbase = Amount(0, market["base"])
    for order in orderbook["asks"]:
        cumsumbase += order["base"]
        cumsumquote += order["quote"]
        ta["asks"].append(
            [
                "{:f} {}/{}".format(
                    order["price"],
                    order["base"]["asset"]["symbol"],
                    order["quote"]["asset"]["symbol"],
                ),
                str(order["base"]),
                str(cumsumbase),
                str(order["quote"]),
                str(cumsumquote),
            ]
        )
    t = [["bids", "asks"]]
    t.append([format_table(ta["bids"]), format_table(ta["asks"])])
    print_table(t)

Response: Show the orderbook of a particular market


Instruction:
Can you generate a brief explanation for the following Python 3 code
def buy(ctx, buy_amount, buy_asset, price, sell_asset, order_expiration, account):
    amount = Amount(buy_amount, buy_asset)
    price = Price(
        price, base=sell_asset, quote=buy_asset, bitshares_instance=ctx.bitshares
    )
    print_tx(
        price.market.buy(price, amount, account=account, expiration=order_expiration)
    )

Response: Buy a specific asset at a certain rate against a base asset


Instruction:
Write a Python 3 function that can
list open orders of an account

Response:
def openorders(ctx, account):
    """ List open orders of an account
    """
    account = Account(
        account or config["default_account"], bitshares_instance=ctx.bitshares
    )
    t = [["Price", "Quote", "Base", "ID"]]
    for o in account.openorders:
        t.append(
            [
                "{:f} {}/{}".format(
                    o["price"],
                    o["base"]["asset"]["symbol"],
                    o["quote"]["asset"]["symbol"],
                ),
                str(o["quote"]),
                str(o["base"]),
                o["id"],
            ]
        )
    print_table(t)


Instruction:
How would you explain what the following Python 3 function does
def cancelall(ctx, market, account):
    market = Market(market)
    ctx.bitshares.bundle = True
    market.cancel([x["id"] for x in market.accountopenorders(account)], account=account)
    print_tx(ctx.bitshares.txbuffer.broadcast())

Response: Cancel all open orders of an account in a market


Instruction:
How would you explain what the following Python 3 function does
def spread(ctx, market, side, min, max, num, total, order_expiration, account):
    from tqdm import tqdm
    from numpy import linspace

    market = Market(market)
    ctx.bitshares.bundle = True

    if min < max:
        space = linspace(min, max, num)
    else:
        space = linspace(max, min, num)

    func = getattr(market, side)
    for p in tqdm(space):
        func(p, total / float(num), account=account, expiration=order_expiration)
    print_tx(ctx.bitshares.txbuffer.broadcast())

Response: Place multiple orders at a given time period


Instruction:
Write a Python 3 function that can
borrow a bitasset/market-pegged asset

Response:
def borrow(ctx, amount, symbol, ratio, account):
    """ Borrow a bitasset/market-pegged asset
    """
    from bitshares.dex import Dex

    dex = Dex(bitshares_instance=ctx.bitshares)
    print_tx(
        dex.borrow(Amount(amount, symbol), collateral_ratio=ratio, account=account)
    )


Instruction:
How would you code a function in Python 3 to
update the collateral ratio of a call positions

Response:
def updateratio(ctx, symbol, ratio, account):
    """ Update the collateral ratio of a call positions
    """
    from bitshares.dex import Dex

    dex = Dex(bitshares_instance=ctx.bitshares)
    print_tx(dex.adjust_collateral_ratio(symbol, ratio, account=account))


Instruction:
Can you write a function in Python 3 where it
funds the fee pool of an asset

Response:
def fundfeepool(ctx, symbol, amount, account):
    """ Fund the fee pool of an asset
    """
    print_tx(ctx.bitshares.fund_fee_pool(symbol, amount, account=account))


Instruction:
Can you implement a function in Python 3 that
bids a collateral in the settlement fund

Response:
def bidcollateral(
    ctx, collateral_symbol, collateral_amount, debt_symbol, debt_amount, account
):
    """ Bid for collateral in the settlement fund
    """
    print_tx(
        ctx.bitshares.bid_collateral(
            Amount(collateral_amount, collateral_symbol),
            Amount(debt_amount, debt_symbol),
            account=account,
        )
    )


Instruction:
Explain what the following Python 3 code does
def settle(ctx, symbol, amount, account):
    print_tx(ctx.bitshares.asset_settle(Amount(amount, symbol), account=account))

Response: Fund the fee pool of an asset


Instruction:
Write a Python 3 script to
list accounts vesting balances

Response:
def votes(ctx, account, type):
    """ List accounts vesting balances
    """
    if not isinstance(type, (list, tuple)):
        type = [type]
    account = Account(account, full=True)
    ret = {key: list() for key in Vote.types()}
    for vote in account["votes"]:
        t = Vote.vote_type_from_id(vote["id"])
        ret[t].append(vote)

    t = [["id", "url", "account"]]
    for vote in ret["committee"]:
        t.append(
            [vote["id"], vote["url"], Account(vote["committee_member_account"])["name"]]
        )

    if "committee" in type:
        t = [["id", "url", "account", "votes"]]
        for vote in ret["committee"]:
            t.append(
                [
                    vote["id"],
                    vote["url"],
                    Account(vote["committee_member_account"])["name"],
                    str(Amount({"amount": vote["total_votes"], "asset_id": "1.3.0"})),
                ]
            )
        print_table(t)

    if "witness" in type:
        t = [
            [
                "id",
                "account",
                "url",
                "votes",
                "last_confirmed_block_num",
                "total_missed",
                "westing",
            ]
        ]
        for vote in ret["witness"]:
            t.append(
                [
                    vote["id"],
                    Account(vote["witness_account"])["name"],
                    vote["url"],
                    str(Amount({"amount": vote["total_votes"], "asset_id": "1.3.0"})),
                    vote["last_confirmed_block_num"],
                    vote["total_missed"],
                    str(Vesting(vote.get("pay_vb")).claimable)
                    if vote.get("pay_vb")
                    else "",
                ]
            )
        print_table(t)

    if "worker" in type:
        t = [["id", "name/url", "daily_pay", "votes", "time", "account"]]
        for vote in ret["worker"]:
            votes = Amount({"amount": vote["total_votes_for"], "asset_id": "1.3.0"})
            amount = Amount({"amount": vote["daily_pay"], "asset_id": "1.3.0"})
            t.append(
                [
                    vote["id"],
                    "{name}\n{url}".format(**vote),
                    str(amount),
                    str(votes),
                    "{work_begin_date}\n-\n{work_end_date}".format(**vote),
                    str(Account(vote["worker_account"])["name"]),
                ]
            )
        print_table(t)


Instruction:
Can you generate the documentation for the following Python 3 function
def info(ctx, objects):
    if not objects:
        t = [["Key", "Value"]]
        info = ctx.bitshares.rpc.get_dynamic_global_properties()
        for key in info:
            t.append([key, info[key]])
        print_table(t)

    for obj in objects:
        # Block
        if re.match("^[0-9]*$", obj):
            block = Block(obj, lazy=False, bitshares_instance=ctx.bitshares)
            t = [["Key", "Value"]]
            for key in sorted(block):
                value = block[key]
                if key == "transactions":
                    value = format_tx(value)
                t.append([key, value])
            print_table(t)
        # Object Id
        elif re.match("^\d*\.\d*\.\d*$", obj):
            data = ctx.bitshares.rpc.get_object(obj)
            if data:
                t = [["Key", "Value"]]
                for key in sorted(data):
                    value = data[key]
                    if isinstance(value, dict) or isinstance(value, list):
                        value = format_tx(value)
                    t.append([key, value])
                print_table(t)
            else:
                print_message("Object %s unknown" % obj, "warning")

        # Asset
        elif obj.upper() == obj and re.match("^[A-Z\.]*$", obj):
            data = Asset(obj)
            t = [["Key", "Value"]]
            for key in sorted(data):
                value = data[key]
                if isinstance(value, dict):
                    value = format_tx(value)
                t.append([key, value])
            print_table(t)

        # Public Key
        elif re.match("^BTS.{48,55}$", obj):
            account = ctx.bitshares.wallet.getAccountFromPublicKey(obj)
            if account:
                t = [["Account"]]
                t.append([account])
                print_table(t)
            else:
                print_message("Public Key not known: %s" % obj, "warning")

        # Account name
        elif re.match("^[a-zA-Z0-9\-\._]{2,64}$", obj):
            account = Account(obj, full=True)
            if account:
                t = [["Key", "Value"]]
                for key in sorted(account):
                    value = account[key]
                    if isinstance(value, dict) or isinstance(value, list):
                        value = format_tx(value)
                    t.append([key, value])
                print_table(t)
            else:
                print_message("Account %s unknown" % obj, "warning")

        elif ":" in obj:
            vote = ctx.bitshares.rpc.lookup_vote_ids([obj])[0]
            if vote:
                t = [["Key", "Value"]]
                for key in sorted(vote):
                    value = vote[key]
                    if isinstance(value, dict) or isinstance(value, list):
                        value = format_tx(value)
                    t.append([key, value])
                print_table(t)
            else:
                print_message("voteid %s unknown" % obj, "warning")

        else:
            print_message("Couldn't identify object to read", "warning")

Response: Obtain all kinds of information for a list of objects.


Instruction:
Can you generate the documentation for the following Python 3 function
def fees(ctx, currency):
    from bitsharesbase.operationids import getOperationNameForId
    from bitshares.market import Market

    market = Market("%s:%s" % (currency, "BTS"))
    ticker = market.ticker()
    if "quoteSettlement_price" in ticker:
        price = ticker.get("quoteSettlement_price")
    else:
        price = ticker.get("latest", 0)
    price.invert()

    chain = Blockchain(bitshares_instance=ctx.bitshares)
    feesObj = chain.chainParameters().get("current_fees")
    fees = feesObj["parameters"]

    t = [["Operation", "Type", "Fee", currency]]

    for fee in fees:
        for f in fee[1]:
            t.append(
                [
                    highlight(getOperationNameForId(fee[0])),
                    detail(f),
                    detail(
                        str(Amount({"amount": fee[1].get(f, 0), "asset_id": "1.3.0"}))
                    ),
                    detail(
                        str(
                            price
                            * Amount({"amount": fee[1].get(f, 0), "asset_id": "1.3.0"})
                        )
                    ),
                ]
            )
    print_table(t)

Response: List fees in a single market


Instruction:
How would you explain what the following Python 3 function does
def create(ctx, to, amount, symbol, secret, hash, account, expiration):
    ctx.blockchain.blocking = True
    tx = ctx.blockchain.htlc_create(
        Amount(amount, symbol),
        to,
        secret,
        hash_type=hash,
        expiration=expiration,
        account=account,
    )
    tx.pop("trx", None)
    print_tx(tx)
    results = tx.get("operation_results", {})
    if results:
        htlc_id = results[0][1]
        print("Your htlc_id is: {}".format(htlc_id))

Response: Create an HTLC contract


Instruction:
Explain what the following Python 3 code does
def redeem(ctx, htlc_id, secret, account):
    print_tx(ctx.blockchain.htlc_redeem(htlc_id, secret, account=account))

Response: Redeem an HTLC contract


Instruction:
Can you generate the documentation for the following Python 3 function
def unique_bits(flags_class):
    flags_class = unique(flags_class)
    other_bits = 0
    for name, member in flags_class.__members_without_aliases__.items():
        bits = int(member)
        if other_bits & bits:
            for other_name, other_member in flags_class.__members_without_aliases__.items():
                if int(other_member) & bits:
                    raise ValueError("%r: '%s' and '%s' have overlapping bits" % (flags_class, other_name, name))
        else:
            other_bits |= bits

Response: A decorator for flags classes to forbid declaring flags with overlapping bits.


Instruction:
Here you have a function in Python 3, explain what it does
def process_inline_members_definition(members):
    if isinstance(members, str):
        members = ((name, UNDEFINED) for name in members.replace(',', ' ').split())
    elif isinstance(members, (tuple, list, collections.Set)):
        if members and isinstance(next(iter(members)), str):
            members = ((name, UNDEFINED) for name in members)
    elif isinstance(members, collections.Mapping):
        members = members.items()
    return members

Response: Process inline members definition.


Instruction:
Create a Python 3 function for
taking only the output of to_simple_str () and returns a new object.

Response:
def from_simple_str(cls, s):
        """ Accepts only the output of to_simple_str(). The output of __str__() is invalid as input. """
        if not isinstance(s, str):
            raise TypeError("Expected an str instance, received %r" % (s,))
        return cls(cls.bits_from_simple_str(s))


Instruction:
Given the following Python 3 function, write the documentation
def from_str(cls, s):
        if not isinstance(s, str):
            raise TypeError("Expected an str instance, received %r" % (s,))
        return cls(cls.bits_from_str(s))

Response: Returns a new instance of the class from a string.