Instruction:
Can you create a Python 3 function that
adds main function gsea argument parsers.

Response:
def add_gsea_parser(subparsers):
    """Add main function 'gsea' argument parsers."""

    argparser_gsea = subparsers.add_parser("gsea", help="Main GSEApy Function: run GSEApy instead of GSEA.")

    # group for input files
    group_input = argparser_gsea.add_argument_group("Input files arguments")
    group_input.add_argument("-d", "--data", dest="data", action="store", type=str, required=True,
                             help="Input gene expression dataset file in txt format.Same with GSEA.")
    group_input.add_argument("-c", "--cls", dest="cls", action="store", type=str, required=True,
                             help="Input class vector (phenotype) file in CLS format. Same with GSEA.")
    group_input.add_argument("-g", "--gmt", dest="gmt", action="store", type=str, required=True,
                             help="Gene set database in GMT format. Same with GSEA.")
    group_input.add_argument("-t", "--permu-type", action="store", dest="type", type=str, metavar='perType',
                             choices=("gene_set", "phenotype"), default="gene_set",
                             help="Permutation type. Same with GSEA, choose from {'gene_set', 'phenotype'}")

    # group for output files
    group_output = argparser_gsea.add_argument_group("Output arguments")
    add_output_option(group_output)

    # group for General options.
    group_opt = argparser_gsea.add_argument_group("GSEA advanced arguments")
    group_opt.add_argument("-n", "--permu-num", dest = "n", action="store", type=int, default=1000, metavar='nperm',
                           help="Number of random permutations. For calculating esnulls. Default: 1000")
    group_opt.add_argument("--min-size",  dest="mins", action="store", type=int, default=15, metavar='int',
                           help="Min size of input genes presented in Gene Sets. Default: 15")
    group_opt.add_argument("--max-size", dest = "maxs", action="store", type=int, default=500, metavar='int',
                           help="Max size of input genes presented in Gene Sets. Default: 500")
    group_opt.add_argument("-w", "--weight", action='store', dest='weight', default=1.0, type=float, metavar='float',
                           help='Weighted_score of rank_metrics. For weighting input genes. Choose from {0, 1, 1.5, 2}. Default: 1',)
    group_opt.add_argument("-m", "--method", action="store", dest="method", type=str, metavar='',
                           choices=("signal_to_noise", "t_test", "ratio_of_classes", "diff_of_classes", "log2_ratio_of_classes"),
                           default="log2_ratio_of_classes",
                           help="Methods to calculate correlations of ranking metrics. \
                           Choose from {'signal_to_noise', 't_test', 'ratio_of_classes', 'diff_of_classes','log2_ratio_of_classes'}.\
                           Default: 'log2_ratio_of_classes'")
    group_opt.add_argument("-a", "--ascending", action='store_true', dest='ascending', default=False,
                           help='Rank metric sorting order. If the -a flag was chosen, then ascending equals to True. Default: False.')
    group_opt.add_argument("-s", "--seed", dest = "seed", action="store", type=int, default=None, metavar='',
                           help="Number of random seed. Default: None")
    group_opt.add_argument("-p", "--threads", dest = "threads", action="store", type=int, default=1, metavar='procs',
                           help="Number of Processes you are going to use. Default: 1")

    return


Instruction:
Given the following Python 3 function, write the documentation
def add_prerank_parser(subparsers):

    argparser_prerank = subparsers.add_parser("prerank", help="Run GSEApy Prerank tool on preranked gene list.")

    # group for input files
    prerank_input = argparser_prerank.add_argument_group("Input files arguments")
    prerank_input.add_argument("-r", "--rnk", dest="rnk", action="store", type=str, required=True,
                             help="Ranking metric file in .rnk format. Same with GSEA.")
    prerank_input.add_argument("-g", "--gmt", dest="gmt", action="store", type=str, required=True,
                             help="Gene set database in GMT format. Same with GSEA.")
    prerank_input.add_argument("-l", "--label", action='store', nargs=2, dest='label',
                             metavar=('pos', 'neg'), type=str, default=('Pos','Neg'),
                             help="The phenotype label argument need two parameters to define. Default: ('Pos','Neg')")

    # group for output files
    prerank_output = argparser_prerank.add_argument_group("Output arguments")
    add_output_option(prerank_output)

    # group for General options.
    prerank_opt = argparser_prerank.add_argument_group("GSEA advanced arguments")
    prerank_opt.add_argument("-n", "--permu-num", dest = "n", action="store", type=int, default=1000, metavar='nperm',
                             help="Number of random permutations. For calculating esnulls. Default: 1000")
    prerank_opt.add_argument("--min-size",  dest="mins", action="store", type=int, default=15, metavar='int',
                             help="Min size of input genes presented in Gene Sets. Default: 15")
    prerank_opt.add_argument("--max-size", dest = "maxs", action="store", type=int, default=500, metavar='int',
                             help="Max size of input genes presented in Gene Sets. Default: 500")
    prerank_opt.add_argument("-w", "--weight", action='store', dest='weight', default=1.0, type=float, metavar='float',
                             help='Weighted_score of rank_metrics. For weighting input genes. Choose from {0, 1, 1.5, 2}. Default: 1',)
    prerank_opt.add_argument("-a", "--ascending", action='store_true', dest='ascending', default=False,
                             help='Rank metric sorting order. If the -a flag was chosen, then ascending equals to True. Default: False.')
    prerank_opt.add_argument("-s", "--seed", dest = "seed", action="store", type=int, default=None, metavar='',
                             help="Number of random seed. Default: None")
    prerank_opt.add_argument("-p", "--threads", dest = "threads", action="store", type=int, default=1, metavar='procs',
                           help="Number of Processes you are going to use. Default: 1")

    return

Response: Add function prerank argument parsers.


Instruction:
Implement a Python 3 function for
adding function plot argument parsers.

Response:
def add_plot_parser(subparsers):
    """Add function 'plot' argument parsers."""

    argparser_replot = subparsers.add_parser("replot", help="Reproduce GSEA desktop output figures.")

    group_replot = argparser_replot.add_argument_group("Input arguments")

    group_replot.add_argument("-i", "--indir", action="store", dest="indir", required=True, metavar='GSEA_dir',
                              help="The GSEA desktop results directroy that you want to reproduce the figure ")
    add_output_option(group_replot)
    #add_output_group( argparser_plot )
    group_replot.add_argument("-w", "--weight", action='store', dest='weight', default=1.0, type=float, metavar='float',
                              help='Weighted_score of rank_metrics. Please Use the same value in GSEA. Choose from (0, 1, 1.5, 2),default: 1',)

    return


Instruction:
Explain what the following Python 3 code does
def add_enrichr_parser(subparsers):

    argparser_enrichr = subparsers.add_parser("enrichr", help="Using Enrichr API to perform GO analysis.")

    # group for required options.
    enrichr_opt = argparser_enrichr.add_argument_group("Input arguments")
    enrichr_opt.add_argument("-i", "--input-list", action="store", dest="gene_list", type=str, required=True, metavar='IDs',
                              help="Enrichr uses a list of gene names as input.")
    enrichr_opt.add_argument("-g", "--gene-sets", action="store", dest="library", type=str, required=True, metavar='GMT',
                              help="Enrichr library name(s) required. Separate each name by comma.")
    enrichr_opt.add_argument("--org", "--organism", action="store", dest="organism", type=str, default='',
                             help="Enrichr supported organism name. Default: human. See here: https://amp.pharm.mssm.edu/modEnrichr.")
    enrichr_opt.add_argument("--ds", "--description", action="store", dest="descrip", type=str, default='enrichr', metavar='STRING',
                              help="It is recommended to enter a short description for your list so that multiple lists \
                              can be differentiated from each other if you choose to save or share your list.")
    enrichr_opt.add_argument("--cut", "--cut-off", action="store", dest="thresh", metavar='float', type=float, default=0.05,
                              help="Adjust-Pval cutoff, used for generating plots. Default: 0.05.")
    enrichr_opt.add_argument("--bg", "--background", action="store", dest="bg", default='hsapiens_gene_ensembl', metavar='BGNUM',
                              help="BioMart Dataset name or Background total genes number. Default: None")
    enrichr_opt.add_argument("-t", "--top-term", dest="term", action="store", type=int, default=10, metavar='int',
                              help="Numbers of top terms shown in the plot. Default: 10")
    # enrichr_opt.add_argument("--scale", dest = "scale", action="store", type=float, default=0.5, metavar='float',
    #                          help="scatter dot scale in the dotplot. Default: 0.5")
    # enrichr_opt.add_argument("--no-plot", action='store_true', dest='no_plot', default=False,
    #                           help="Suppress the plot output.This is useful only if data are interested. Default: False.")

    enrichr_output = argparser_enrichr.add_argument_group("Output figure arguments")
    add_output_option(enrichr_output)
    return

Response: Add function enrichr argument parsers.


Instruction:
How would you explain what the following Python 3 function does
def add_biomart_parser(subparsers):

    argparser_biomart = subparsers.add_parser("biomart", help="Using BioMart API to convert gene ids.")

    # group for required options.
    biomart_opt = argparser_biomart.add_argument_group("Input arguments")
    biomart_opt.add_argument("-f", "--filter", action='store', nargs=2, dest='filter',
                             required=True, metavar=('NAME', 'VALUE'),
                             help="""Which filter to use. Input filter name, and value.
                                     If multi-value required, separate each value by comma.
                                     If value is a txt file, then one ID per row, exclude header.""")
    biomart_opt.add_argument("-a", "--attributes", action="store", dest="attrs", type=str, required=True, metavar='ATTR',
                             help="Which attribute(s) to retrieve. Separate each attr by comma.")
    biomart_opt.add_argument("-o", "--ofile", dest="ofile", type=str, required=True, help="Output file name")
    biomart_opt.add_argument("-d", "--dataset", action="store", dest="bg", type=str, default='hsapiens_gene_ensembl', metavar='DATA',
                             help="Which dataset to use. Default: hsapiens_gene_ensembl")
    biomart_opt.add_argument("--host", action="store", dest="host", type=str, default='www.ensembl.org', metavar='HOST',
                             help="Which host to use. Select from {'www.ensembl.org', 'asia.ensembl.org', 'useast.ensembl.org'}.")
    biomart_opt.add_argument("-m", "--mart", action="store", dest="mart", type=str, metavar='MART',
                             default="ENSEMBL_MART_ENSEMBL", help="Which mart to use. Default: ENSEMBL_MART_ENSEMBL.")
    biomart_opt.add_argument("-v", "--verbose", action="store_true", default=False, dest='verbose',
                             help="Increase output verbosity, print out progress of your job", )

Response: Add function biomart argument parsers.


Instruction:
How would you implement a function in Python 3 that
generates a tensor that contains the score of each location in the gene list.

Response:
def enrichment_score_tensor(gene_mat, cor_mat, gene_sets, weighted_score_type, nperm=1000,
                            rs=np.random.RandomState(), single=False, scale=False):
    """Next generation algorithm of GSEA and ssGSEA.

        :param gene_mat:        the ordered gene list(vector) with or without gene indices matrix.
        :param cor_mat:         correlation vector or matrix  (e.g. signal to noise scores)
                                corresponding to the genes in the gene list or matrix.
        :param dict gene_sets:  gmt file dict.
        :param float weighted_score_type:     weighting by the correlation.
                                options: 0(classic), 1, 1.5, 2. default:1 for GSEA and 0.25 for ssGSEA.
        :param int nperm:       permutation times.
        :param bool scale:      If True, normalize the scores by number of genes_mat.
        :param bool single:     If True, use ssGSEA algorithm, otherwise use GSEA.
        :param rs:              Random state for initialize gene list shuffling.
                                Default: np.random.RandomState(seed=None)
        :return: a tuple contains::

                 | ES: Enrichment score (real number between -1 and +1), for ssGSEA, set scale eq to True.
                 | ESNULL: Enrichment score calculated from random permutation.
                 | Hits_Indices: Indices of genes if genes are included in gene_set.
                 | RES: The running enrichment score for all locations in the gene list.

    """
    # gene_mat -> 1d: prerank, ssSSEA or 2d: GSEA
    keys = sorted(gene_sets.keys())

    if weighted_score_type == 0:
        # don't bother doing calcuation, just set to 1
        cor_mat = np.ones(cor_mat.shape)
    elif weighted_score_type > 0:
        pass
    else:
        logging.error("Using negative values of weighted_score_type, not allowed")
        sys.exit(0)

    cor_mat = np.abs(cor_mat)
    if cor_mat.ndim ==1:
        # ssGSEA or Prerank
        # genestes->M, genes->N, perm-> axis=2
        N, M = len(gene_mat), len(keys)
        # generate gene hits matrix
        # for 1d ndarray of gene_mat, set assume_unique=True,
        # means the input arrays are both assumed to be unique,
        # which can speed up the calculation.
        tag_indicator = np.vstack([np.in1d(gene_mat, gene_sets[key], assume_unique=True) for key in keys])
        tag_indicator = tag_indicator.astype(int)
        # index of hits
        hit_ind = [ np.flatnonzero(tag).tolist() for tag in tag_indicator ]
        # generate permutated hits matrix
        perm_tag_tensor = np.repeat(tag_indicator, nperm+1).reshape((M,N,nperm+1))
        # shuffle matrix, last matrix is not shuffled when nperm > 0
        if nperm: np.apply_along_axis(lambda x: np.apply_along_axis(rs.shuffle,0,x),1, perm_tag_tensor[:,:,:-1])
        # missing hits
        no_tag_tensor = 1 - perm_tag_tensor
        # calculate numerator, denominator of each gene hits
        rank_alpha = (perm_tag_tensor*cor_mat[np.newaxis,:,np.newaxis])** weighted_score_type

    elif cor_mat.ndim == 2:
        # GSEA
        # 2d ndarray, gene_mat and cor_mat are shuffled already
        # reshape matrix
        cor_mat = cor_mat.T
        # gene_mat is a tuple contains (gene_name, permuate_gene_name_indices)
        genes, genes_ind = gene_mat
        # genestes->M, genes->N, perm-> axis=2
        # don't use assume_unique=True in 2d array when use np.isin().
        # elements in gene_mat are not unique, or will cause unwanted results
        tag_indicator = np.vstack([np.in1d(genes, gene_sets[key], assume_unique=True) for key in keys])
        tag_indicator = tag_indicator.astype(int)
        perm_tag_tensor = np.stack([tag.take(genes_ind).T for tag in tag_indicator], axis=0)
        #index of hits
        hit_ind = [ np.flatnonzero(tag).tolist() for tag in perm_tag_tensor[:,:,-1] ]
        # nohits
        no_tag_tensor = 1 - perm_tag_tensor
        # calculate numerator, denominator of each gene hits
        rank_alpha = (perm_tag_tensor*cor_mat[np.newaxis,:,:])** weighted_score_type
    else:
        logging.error("Program die because of unsupported input")
        sys.exit(0)

    # Nhint = tag_indicator.sum(1)
    # Nmiss =  N - Nhint
    axis=1
    P_GW_denominator = np.sum(rank_alpha, axis=axis, keepdims=True)
    P_NG_denominator = np.sum(no_tag_tensor, axis=axis, keepdims=True)
    REStensor = np.cumsum(rank_alpha / P_GW_denominator - no_tag_tensor / P_NG_denominator, axis=axis)
    # ssGSEA: scale es by gene numbers ?
    # https://gist.github.com/gaoce/39e0907146c752c127728ad74e123b33
    if scale: REStensor = REStensor / len(gene_mat)
    if single:
        #ssGSEA
        esmatrix = REStensor.sum(axis=axis)
    else:
        #GSEA
        esmax, esmin = REStensor.max(axis=axis), REStensor.min(axis=axis)
        esmatrix = np.where(np.abs(esmax)>np.abs(esmin), esmax, esmin)

    es, esnull, RES = esmatrix[:,-1], esmatrix[:,:-1], REStensor[:,:,-1]

    return es, esnull, hit_ind, RES


Instruction:
Given the following Python 3 function, write the documentation
def ranking_metric_tensor(exprs, method, permutation_num, pos, neg, classes,
                          ascending, rs=np.random.RandomState()):
    """Build shuffled ranking matrix when permutation_type eq to phenotype.

       :param exprs:   gene_expression DataFrame, gene_name indexed.
       :param str method:  calculate correlation or ranking. methods including:
                           1. 'signal_to_noise'.
                           2. 't_test'.
                           3. 'ratio_of_classes' (also referred to as fold change).
                           4. 'diff_of_classes'.
                           5. 'log2_ratio_of_classes'.
       :param int permuation_num: how many times of classes is being shuffled
       :param str pos: one of labels of phenotype's names.
       :param str neg: one of labels of phenotype's names.
       :param list classes:  a list of phenotype labels, to specify which column of
                             dataframe belongs to what class of phenotype.
       :param bool ascending:  bool. Sort ascending vs. descending.

       :return:
                returns two 2d ndarray with shape (nperm, gene_num).

                | cor_mat_indices: the indices of sorted and permutated (exclude last row) ranking matrix.
                | cor_mat: sorted and permutated (exclude last row) ranking matrix.

    """
    # S: samples, G: gene number
    G, S = exprs.shape
    # genes = exprs.index.values
    expr_mat = exprs.values.T
    perm_cor_tensor = np.tile(expr_mat, (permutation_num+1,1,1))
    # random shuffle on the first dim, last matrix is not shuffled
    for arr in perm_cor_tensor[:-1]: rs.shuffle(arr)
    classes = np.array(classes)
    pos = classes == pos
    neg = classes == neg
    pos_cor_mean = perm_cor_tensor[:,pos,:].mean(axis=1)
    neg_cor_mean = perm_cor_tensor[:,neg,:].mean(axis=1)
    pos_cor_std = perm_cor_tensor[:,pos,:].std(axis=1, ddof=1)
    neg_cor_std = perm_cor_tensor[:,neg,:].std(axis=1, ddof=1)

    if method == 'signal_to_noise':
        cor_mat = (pos_cor_mean - neg_cor_mean)/(pos_cor_std + neg_cor_std)
    elif method == 't_test':
        denom = 1.0/G
        cor_mat = (pos_cor_mean - neg_cor_mean)/ np.sqrt(denom*pos_cor_std**2 + denom*neg_cor_std**2)
    elif method == 'ratio_of_classes':
        cor_mat = pos_cor_mean / neg_cor_mean
    elif method == 'diff_of_classes':
        cor_mat  = pos_cor_mean - neg_cor_mean
    elif method == 'log2_ratio_of_classes':
        cor_mat  =  np.log2(pos_cor_mean / neg_cor_mean)
    else:
        logging.error("Please provide correct method name!!!")
        sys.exit(0)
    # return matix[nperm+1, perm_cors]
    cor_mat_ind = cor_mat.argsort()
    # ndarray: sort in place
    cor_mat.sort()
    # genes_mat = genes.take(cor_mat_ind)
    if ascending: return cor_mat_ind, cor_mat
    # descending order of ranking and genes
    # return genes_mat[:,::-1], cor_mat[:,::-1]
    return cor_mat_ind[:, ::-1], cor_mat[:, ::-1]

Response: Builds a ranking metric tensor for a given set of genes.


Instruction:
Make a summary of the following Python 3 code
def ranking_metric(df, method, pos, neg, classes, ascending):

    # exclude any zero stds.
    df_mean = df.groupby(by=classes, axis=1).mean()
    df_std =  df.groupby(by=classes, axis=1).std()

    if method == 'signal_to_noise':
        ser = (df_mean[pos] - df_mean[neg])/(df_std[pos] + df_std[neg])
    elif method == 't_test':
        ser = (df_mean[pos] - df_mean[neg])/ np.sqrt(df_std[pos]**2/len(df_std)+df_std[neg]**2/len(df_std) )
    elif method == 'ratio_of_classes':
        ser = df_mean[pos] / df_mean[neg]
    elif method == 'diff_of_classes':
        ser  = df_mean[pos] - df_mean[neg]
    elif method == 'log2_ratio_of_classes':
        ser  =  np.log2(df_mean[pos] / df_mean[neg])
    else:
        logging.error("Please provide correct method name!!!")
        sys.exit(0)
    ser = ser.sort_values(ascending=ascending)

    return ser

Response: This function is used to calculate a metric for a given gene expression table.


Instruction:
Implement a Python 3 function for
computing the ranked terms and ranked scores for each gene set in the input dataframe

Response:
def gsea_compute_tensor(data, gmt, n, weighted_score_type, permutation_type,
                 method, pheno_pos, pheno_neg, classes, ascending,
                 processes=1, seed=None, single=False, scale=False):
    """compute enrichment scores and enrichment nulls.

        :param data: preprocessed expression dataframe or a pre-ranked file if prerank=True.
        :param dict gmt: all gene sets in .gmt file. need to call load_gmt() to get results.
        :param int n: permutation number. default: 1000.
        :param str method: ranking_metric method. see above.
        :param str pheno_pos: one of labels of phenotype's names.
        :param str pheno_neg: one of labels of phenotype's names.
        :param list classes: a list of phenotype labels, to specify which column of dataframe belongs to what category of phenotype.
        :param float weighted_score_type: default:1
        :param bool ascending: sorting order of rankings. Default: False.
        :param seed: random seed. Default: np.random.RandomState()
        :param bool scale: if true, scale es by gene number.

        :return: a tuple contains::

                | zipped results of es, nes, pval, fdr.
                | nested list of hit indices of input gene_list.
                | nested list of ranked enrichment score of each input gene_sets.
                | list of enriched terms

    """
    w = weighted_score_type
    subsets = sorted(gmt.keys())
    rs = np.random.RandomState(seed)
    genes_mat, cor_mat = data.index.values, data.values
    base = 5 if data.shape[0] >= 5000 else 10
    block = ceil(len(subsets) / base)

    if permutation_type == "phenotype":
        # shuffling classes and generate random correlation rankings
        logging.debug("Start to permutate classes..............................")
        genes_ind = []
        cor_mat = []
        temp_rnk = []
        pool_rnk = Pool(processes=processes)
        # split large array into smaller blocks to avoid memory overflow
        i=1
        while i <= block:
            rs = np.random.RandomState(seed)
            temp_rnk.append(pool_rnk.apply_async(ranking_metric_tensor,
                                         args=(data, method, base, pheno_pos, pheno_neg, classes,
                                                ascending, rs)))
            i +=1
        pool_rnk.close()
        pool_rnk.join()

        for k, temp in enumerate(temp_rnk):
            gi, cor = temp.get()
            if k+1 == block:
               genes_ind.append(gi)
               cor_mat.append(cor)
            else:
                genes_ind.append(gi[:-1])
                cor_mat.append(cor[:-1])

        genes_ind, cor_mat = np.vstack(genes_ind), np.vstack(cor_mat)
        # convert to tuple
        genes_mat = (data.index.values, genes_ind)

    logging.debug("Start to compute es and esnulls........................")
    # Prerank, ssGSEA, GSEA
    es = []
    RES = []
    hit_ind = []
    esnull = []
    temp_esnu = []
    pool_esnu = Pool(processes=processes)
    # split large array into smaller blocks to avoid memory overflow
    i, m = 1, 0
    while i <= block:
        # you have to reseed, or all your processes are sharing the same seed value
        rs = np.random.RandomState(seed)
        gmtrim = {k: gmt.get(k) for k in subsets[m:base * i]}
        temp_esnu.append(pool_esnu.apply_async(enrichment_score_tensor,
                                               args=(genes_mat, cor_mat,
                                                     gmtrim, w, n, rs,
                                                     single, scale)))
        m = base * i
        i += 1
    pool_esnu.close()
    pool_esnu.join()
    # esn is a list, don't need to use append method.
    for si, temp in enumerate(temp_esnu):
        e, enu, hit, rune = temp.get()
        esnull.append(enu)
        es.append(e)
        RES.append(rune)
        hit_ind += hit
    # concate results
    es, esnull, RES = np.hstack(es), np.vstack(esnull), np.vstack(RES)

    return gsea_significance(es, esnull), hit_ind, RES, subsets


Instruction:
Given the following Python 3 function, write the documentation
def gsea_compute(data, gmt, n, weighted_score_type, permutation_type,
                 method, pheno_pos, pheno_neg, classes, ascending,
                 processes=1, seed=None, single=False, scale=False):
    """compute enrichment scores and enrichment nulls.

        :param data: preprocessed expression dataframe or a pre-ranked file if prerank=True.
        :param dict gmt: all gene sets in .gmt file. need to call load_gmt() to get results.
        :param int n: permutation number. default: 1000.
        :param str method: ranking_metric method. see above.
        :param str pheno_pos: one of labels of phenotype's names.
        :param str pheno_neg: one of labels of phenotype's names.
        :param list classes: a list of phenotype labels, to specify which column of dataframe belongs to what category of phenotype.
        :param float weighted_score_type: default:1
        :param bool ascending: sorting order of rankings. Default: False.
        :param seed: random seed. Default: np.random.RandomState()
        :param bool scale: if true, scale es by gene number.

        :return: a tuple contains::

                | zipped results of es, nes, pval, fdr.
                | nested list of hit indices of input gene_list.
                | nested list of ranked enrichment score of each input gene_sets.
                | list of enriched terms

    """

    w = weighted_score_type
    subsets = sorted(gmt.keys())
    es = []
    RES=[]
    hit_ind=[]
    esnull = [ [] for a in range(len(subsets)) ]

    logging.debug("Start to compute enrichment scores......................")

    if permutation_type == "phenotype":
        logging.debug("Start to permutate classes..............................")
        # shuffling classes and generate random correlation rankings
        rs = np.random.RandomState(seed)
        genes_mat, cor_mat = ranking_metric_tensor(exprs=data, method=method,
                                                   permutation_num=n,
                                                   pos=pheno_pos, neg=pheno_neg,
                                                   classes=classes,
                                                   ascending=ascending, rs=rs)

        # compute es, esnulls. hits, RES
        logging.debug("Start to compute enrichment nulls.......................")
        es, esnull, hit_ind, RES = enrichment_score_tensor(gene_mat=genes_mat,
                                                           cor_mat=cor_mat,
                                                           gene_sets=gmt,
                                                           weighted_score_type=w,
                                                           nperm=n, rs=rs,
                                                           single=False, scale=False,)

    else:
        # Prerank, ssGSEA, GSEA with gene_set permutation
        gl, cor_vec = data.index.values, data.values
        logging.debug("Start to compute es and esnulls........................")
        # es, esnull, hit_ind, RES = enrichment_score_tensor(gene_mat=gl,
        #                                                    cor_mat=cor_mat,
        #                                                    gene_sets=gmt,
        #                                                    weighted_score_type=w,
        #                                                    nperm=n, rs=rs
        #                                                    single=single, scale=scale)

        # split large array into smaller blocks to avoid memory overflow
        temp_esnu=[]
        pool_esnu = Pool(processes=processes)
        for subset in subsets:
            # you have to reseed, or all your processes are sharing the same seed value
            rs = np.random.RandomState(seed)
            temp_esnu.append(pool_esnu.apply_async(enrichment_score,
                                                   args=(gl, cor_vec, gmt.get(subset), w,
                                                         n, rs, single, scale)))

        pool_esnu.close()
        pool_esnu.join()
        # esn is a list, don't need to use append method.
        for si, temp in enumerate(temp_esnu):
            e, enu, hit, rune = temp.get()
            esnull[si] = enu
            es.append(e)
            RES.append(rune)
            hit_ind.append(hit)

    return gsea_significance(es, esnull), hit_ind, RES, subsets

Response: compute the ranked terms and null scores for each gene set in the input expression dataframe


Instruction:
Write a Python 3 script to
compute nominal p - value for S given an observed ES and an empty array.

Response:
def gsea_pval(es, esnull):
    """Compute nominal p-value.

    From article (PNAS):
    estimate nominal p-value for S from esnull by using the positive
    or negative portion of the distribution corresponding to the sign
    of the observed ES(S).
    """

    # to speed up, using numpy function to compute pval in parallel.
    condlist = [ es < 0, es >=0]
    choicelist = [np.sum(esnull < es.reshape(len(es),1), axis=1)/ np.sum(esnull < 0, axis=1),
                  np.sum(esnull >= es.reshape(len(es),1), axis=1)/ np.sum(esnull >= 0, axis=1)]
    pval = np.select(condlist, choicelist)

    return pval


Instruction:
Can you write a function in Python 3 where it
computes nominal pvals normalized ES and FDR q value.

Response:
def gsea_significance(enrichment_scores, enrichment_nulls):
    """Compute nominal pvals, normalized ES, and FDR q value.

        For a given NES(S) = NES* >= 0. The FDR is the ratio of the percentage of all (S,pi) with
        NES(S,pi) >= 0, whose NES(S,pi) >= NES*, divided by the percentage of
        observed S wih NES(S) >= 0, whose NES(S) >= NES*, and similarly if NES(S) = NES* <= 0.
    """
    # For a zero by zero division (undetermined, results in a NaN),
    np.seterr(divide='ignore', invalid='ignore')
    # import warnings
    # warnings.simplefilter("ignore")
    es = np.array(enrichment_scores)
    esnull = np.array(enrichment_nulls)
    logging.debug("Start to compute pvals..................................")
    # compute pvals.
    enrichmentPVals = gsea_pval(es, esnull).tolist()

    logging.debug("Compute nes and nesnull.................................")
    # nEnrichmentScores, nEnrichmentNulls = normalize(es, esnull)

    # new normalized enrichment score implementation.
    # this could speed up significantly.
    esnull_pos = (esnull*(esnull>=0)).mean(axis=1)
    esnull_neg = (esnull*(esnull<0)).mean(axis=1)
    nEnrichmentScores  = np.where(es>=0, es/esnull_pos, -es/esnull_neg)
    nEnrichmentNulls = np.where(esnull>=0, esnull/esnull_pos[:,np.newaxis],
                                          -esnull/esnull_neg[:,np.newaxis])

    logging.debug("start to compute fdrs..................................")

    # FDR null distribution histogram
    # create a histogram of all NES(S,pi) over all S and pi
    # Use this null distribution to compute an FDR q value,
    # vals = reduce(lambda x,y: x+y, nEnrichmentNulls, [])
    # nvals = np.array(sorted(vals))
    # or
    nvals = np.sort(nEnrichmentNulls.flatten())
    nnes = np.sort(nEnrichmentScores)
    fdrs = []
    # FDR computation
    for i in range(len(enrichment_scores)):
        nes = nEnrichmentScores[i]
        # use the same pval method to calculate fdr
        if nes >= 0:
            allPos = int(len(nvals) - np.searchsorted(nvals, 0, side="left"))
            allHigherAndPos = int(len(nvals) - np.searchsorted(nvals, nes, side="left"))
            nesPos = len(nnes) - int(np.searchsorted(nnes, 0, side="left"))
            nesHigherAndPos = len(nnes) - int(np.searchsorted(nnes, nes, side="left"))
            # allPos = (nvals >= 0).sum()
            # allHigherAndPos = (nvals >= nes).sum()
            # nesPos = (nnes >=0).sum()
            # nesHigherAndPos = (nnes >= nes).sum()
        else:
            allPos = int(np.searchsorted(nvals, 0, side="left"))
            allHigherAndPos = int(np.searchsorted(nvals, nes, side="right"))
            nesPos = int(np.searchsorted(nnes, 0, side="left"))
            nesHigherAndPos = int(np.searchsorted(nnes, nes, side="right"))
            # allPos = (nvals < 0).sum()
            # allHigherAndPos = (nvals < nes).sum()
            # nesPos = (nnes < 0).sum()
            # nesHigherAndPos = (nnes < nes).sum()

        try:
            pi_norm = allHigherAndPos/float(allPos)
            pi_obs = nesHigherAndPos/float(nesPos)
            fdr =  pi_norm / pi_obs
            fdrs.append(fdr if fdr < 1 else 1.0)
        except:
            fdrs.append(1000000000.0)

    logging.debug("Statistical testing finished.............................")

    return zip(enrichment_scores, nEnrichmentScores, enrichmentPVals, fdrs)


Instruction:
Given the following Python 3 function, write the documentation
def log_init(outlog, log_level=logging.INFO):

    # clear old root logger handlers
    logging.getLogger("gseapy").handlers = []
    # init a root logger
    logging.basicConfig(level    = logging.DEBUG,
                        format   = 'LINE %(lineno)-4d: %(asctime)s [%(levelname)-8s] %(message)s',
                        filename = outlog,
                        filemode = 'w')

    # define a Handler which writes INFO messages or higher to the sys.stderr
    console = logging.StreamHandler()
    console.setLevel(log_level)
    # set a format which is simpler for console use
    formatter = logging.Formatter('%(asctime)s %(message)s')
    # tell the handler to use this format
    console.setFormatter(formatter)
    # add handlers
    logging.getLogger("gseapy").addHandler(console)
    logger = logging.getLogger("gseapy")
    # logger.setLevel(log_level)
    return logger

Response: Initialize the root logger with a new log level.


Instruction:
Write a Python 3 function for
retrying connection. define max tries num define max tries num define max tries num define max tries num define max tries num define max tries num

Response:
def retry(num=5):
    """"retry connection.

        define max tries num
        if the backoff_factor is 0.1, then sleep() will sleep for
        [0.1s, 0.2s, 0.4s, ...] between retries.
        It will also force a retry if the status code returned is 500, 502, 503 or 504.

    """
    s = requests.Session()
    retries = Retry(total=num, backoff_factor=0.1,
                    status_forcelist=[500, 502, 503, 504])
    s.mount('http://', HTTPAdapter(max_retries=retries))

    return s


Instruction:
Can you implement a function in Python 3 that
extracts class name from. cls file.

Response:
def gsea_cls_parser(cls):
    """Extract class(phenotype) name from .cls file.

    :param cls: the a class list instance or .cls file which is identical to GSEA input .
    :return: phenotype name and a list of class vector.
    """

    if isinstance(cls, list) :
        classes = cls
        sample_name= unique(classes)
    elif isinstance(cls, str) :
        with open(cls) as c:
            file = c.readlines()
        classes = file[2].strip('\n').split(" ")
        sample_name = file[1].lstrip("# ").strip('\n').split(" ")
    else:
        raise Exception('Error parsing sample name!')

    return sample_name[0], sample_name[1], classes


Instruction:
Write a Python 3 script to
parse the. results file stored under the GSEA database.

Response:
def gsea_edb_parser(results_path, index=0):
    """Parse results.edb file stored under **edb** file folder.

    :param results_path: the .results file located inside edb folder.
    :param index: gene_set index of gmt database, used for iterating items.
    :return: enrichment_term, hit_index,nes, pval, fdr.
    """
    from bs4 import BeautifulSoup

    soup = BeautifulSoup(open(results_path), features='xml')
    tag = soup.findAll('DTG')
    term = dict(tag[index].attrs)
    # dict_keys(['RANKED_LIST', 'GENESET', 'FWER', 'ES_PROFILE',
    # 'HIT_INDICES', 'ES', 'NES', 'TEMPLATE', 'RND_ES', 'RANK_SCORE_AT_ES',
    # 'NP', 'RANK_AT_ES', 'FDR'])
    enrich_term = term.get('GENESET').split("#")[1]
    es_profile = term.get('ES_PROFILE').split(" ")
    # rank_es = term.get('RND_ES').split(" ")
    hit_ind =term.get('HIT_INDICES').split(" ")
    es_profile = [float(i) for i in es_profile ]
    hit_ind = [float(i) for i in hit_ind ]
    #r ank_es = [float(i) for i in rank_es ]
    nes = term.get('NES')
    pval = term.get('NP')
    fdr =  term.get('FDR')
    # fwer = term.get('FWER')
    # index_range = len(tag)-1
    logging.debug("Enriched Gene set is: "+ enrich_term)

    return enrich_term, hit_ind, nes, pval, fdr


Instruction:
Can you generate a brief explanation for the following Python 3 code
def gsea_gmt_parser(gmt, min_size = 3, max_size = 1000, gene_list=None):

    if gmt.lower().endswith(".gmt"):
        logging.info("User Defined gene sets is given.......continue..........")
        with open(gmt) as genesets:
             genesets_dict = { line.strip().split("\t")[0]: line.strip().split("\t")[2:]
                              for line in genesets.readlines()}
    else:
        logging.info("Downloading and generating Enrichr library gene sets...")
        if gmt in DEFAULT_LIBRARY:
            names = DEFAULT_LIBRARY
        else:
            names = get_library_name()
        if gmt in names:
            """
            define max tries num
            if the backoff_factor is 0.1, then sleep() will sleep for
            [0.1s, 0.2s, 0.4s, ...] between retries.
            It will also force a retry if the status code returned is 500, 502, 503 or 504.
            """
            s = requests.Session()
            retries = Retry(total=5, backoff_factor=0.1,
                            status_forcelist=[ 500, 502, 503, 504 ])
            s.mount('http://', HTTPAdapter(max_retries=retries))
            # queery string
            ENRICHR_URL = 'http://amp.pharm.mssm.edu/Enrichr/geneSetLibrary'
            query_string = '?mode=text&libraryName=%s'
            # get
            response = s.get( ENRICHR_URL + query_string % gmt, timeout=None)
        else:
            raise Exception("gene_set files(.gmt) not found")
        if not response.ok:
            raise Exception('Error fetching enrichment results, check internet connection first.')

        genesets_dict = { line.strip().split("\t")[0]:
                          list(map(lambda x: x.split(",")[0], line.strip().split("\t")[2:]))
                          for line in response.iter_lines(chunk_size=1024, decode_unicode='utf-8')}

    # filtering dict
    if sys.version_info[0] >= 3 :
        genesets_filter =  {k: v for k, v in genesets_dict.items() if len(v) >= min_size and len(v) <= max_size}
    elif sys.version_info[0] == 2:
        genesets_filter =  {k: v for k, v in genesets_dict.iteritems() if len(v) >= min_size and len(v) <= max_size}
    else:
        logging.error("System failure. Please Provide correct input files")
        sys.exit(1)
    if gene_list is not None:
        subsets = sorted(genesets_filter.keys())
        for subset in subsets:
            tag_indicator = in1d(gene_list, genesets_filter.get(subset), assume_unique=True)
            tag_len = sum(tag_indicator)
            if tag_len <= min_size or tag_len >= max_size:
                del genesets_filter[subset]
            else:
                continue
    # some_dict = {key: value for key, value in some_dict.items() if value != value_to_remove}
    # use np.intersect1d() may be faster???
    filsets_num = len(genesets_dict) - len(genesets_filter)
    logging.info("%04d gene_sets have been filtered out when max_size=%s and min_size=%s"%(filsets_num, max_size, min_size))

    if filsets_num == len(genesets_dict):
        logging.error("No gene sets passed throught filtering condition!!!, try new paramters again!\n" +\
                         "Note: Gene names for gseapy is case sensitive." )
        sys.exit(1)
    else:
        return genesets_filter

Response: Parse GSEA GEMT file and return a dictionary of gene sets.


Instruction:
Write a Python 3 script to
get the active enrichr library name.

Response:
def get_library_name(database='Human'):
    """return enrichr active enrichr library name.
    :param str database: Select one from { 'Human', 'Mouse', 'Yeast', 'Fly', 'Fish', 'Worm' }

    """

    # make a get request to get the gmt names and meta data from Enrichr

    # old code
    # response = requests.get('http://amp.pharm.mssm.edu/Enrichr/geneSetLibrary?mode=meta')
    # gmt_data = response.json()
    # # generate list of lib names
    # libs = []
    # # get library names
    # for inst_gmt in gmt_data['libraries']:
    #     # only include active gmts
    #     if inst_gmt['isActive'] == True:
    #         libs.append(inst_gmt['libraryName'])

    if database not in ['Human', 'Mouse', 'Yeast', 'Fly', 'Fish', 'Worm']:
        sys.stderr.write("""No supported database. Please input one of these:
                            'Human', 'Mouse', 'Yeast', 'Fly', 'Fish', 'Worm' """)
        return
    if database in ['Human', 'Mouse']: database=''
    lib_url='http://amp.pharm.mssm.edu/%sEnrichr/datasetStatistics'%database
    libs_json = json.loads(requests.get(lib_url).text)
    libs = [lib['libraryName'] for lib in libs_json['statistics']]

    return sorted(libs)


Instruction:
How would you explain what the following Python 3 function does
def get_marts(self):

        mart_names = pd.Series(self.names, name="Name")
        mart_descriptions = pd.Series(self.displayNames, name="Description")

        return pd.concat([mart_names, mart_descriptions], axis=1)

Response: Get available marts and their names.


Instruction:
How would you explain what the following Python 3 function does
def get_datasets(self, mart='ENSEMBL_MART_ENSEMBL'):
        datasets = self.datasets(mart, raw=True)
        return pd.read_csv(StringIO(datasets), header=None, usecols=[1, 2],
                            names = ["Name", "Description"],sep="\t")

Response: Get available datasets from mart


Instruction:
How would you explain what the following Python 3 function does
def get_attributes(self, dataset):
        attributes = self.attributes(dataset)
        attr_ = [ (k, v[0]) for k, v in attributes.items()]
        return pd.DataFrame(attr_, columns=["Attribute","Description"])

Response: Get available attritbutes from dataset you ve selected


Instruction:
Can you implement a function in Python 3 that
gets available filters from dataset you ve selected

Response:
def get_filters(self, dataset):
        """Get available filters from dataset you've selected"""
        filters = self.filters(dataset)
        filt_ = [ (k, v[0]) for k, v in filters.items()]
        return pd.DataFrame(filt_, columns=["Filter", "Description"])


Instruction:
Can you generate the documentation for the following Python 3 function
def query(self, dataset='hsapiens_gene_ensembl', attributes=[],
              filters={}, filename=None):
        """mapping ids using BioMart.

        :param dataset: str, default: 'hsapiens_gene_ensembl'
        :param attributes: str, list, tuple
        :param filters: dict, {'filter name': list(filter value)}
        :param host: www.ensembl.org, asia.ensembl.org, useast.ensembl.org
        :return: a dataframe contains all attributes you selected.

        **Note**: it will take a couple of minutes to get the results.
        A xml template for querying biomart. (see https://gist.github.com/keithshep/7776579)

        exampleTaxonomy = "mmusculus_gene_ensembl"
        exampleGene = "ENSMUSG00000086981,ENSMUSG00000086982,ENSMUSG00000086983"
        urlTemplate = \
        '''http://ensembl.org/biomart/martservice?query=''' \
        '''<?xml version="1.0" encoding="UTF-8"?>''' \
        '''<!DOCTYPE Query>''' \
        '''<Query virtualSchemaName="default" formatter="CSV" header="0" uniqueRows="0" count="" datasetConfigVersion="0.6">''' \
        '''<Dataset name="%s" interface="default"><Filter name="ensembl_gene_id" value="%s"/>''' \
        '''<Attribute name="ensembl_gene_id"/><Attribute name="ensembl_transcript_id"/>''' \
        '''<Attribute name="transcript_start"/><Attribute name="transcript_end"/>''' \
        '''<Attribute name="exon_chrom_start"/><Attribute name="exon_chrom_end"/>''' \
        '''</Dataset>''' \
        '''</Query>'''

        exampleURL = urlTemplate % (exampleTaxonomy, exampleGene)
        req = requests.get(exampleURL, stream=True)

        """
        if not attributes:
            attributes = ['ensembl_gene_id', 'external_gene_name', 'entrezgene', 'go_id']
        # i=0
        # while (self.host is None) and (i < 3):
        #     self.host = self.ghosts[i]
        #     i +=1
        self.new_query()
        # 'mmusculus_gene_ensembl'
        self.add_dataset_to_xml(dataset)
        for at in attributes:
            self.add_attribute_to_xml(at)
        # add filters
        if filters:
            for k, v in filters.items():
                if isinstance(v, list): v = ",".join(v)
                self.add_filter_to_xml(k, v)

        xml_query = self.get_xml()
        results = super(Biomart, self).query(xml_query)
        df = pd.read_csv(StringIO(results), header=None, sep="\t",
                         names=attributes, index_col=None)
        # save file to cache path.
        if filename is None:
            mkdirs(DEFAULT_CACHE_PATH)
            filename = os.path.join(DEFAULT_CACHE_PATH, "{}.background.genes.txt".format(dataset))
        df.to_csv(filename, sep="\t", index=False)

        return df

Response: Query BioMart for mapping ids using BioMart.


Instruction:
Can you generate the documentation for the following Python 3 function
def gsea(data, gene_sets, cls, outdir='GSEA_', min_size=15, max_size=500, permutation_num=1000,
          weighted_score_type=1,permutation_type='gene_set', method='log2_ratio_of_classes',
	      ascending=False, processes=1, figsize=(6.5,6), format='pdf',
          graph_num=20, no_plot=False, seed=None, verbose=False):
    """ Run Gene Set Enrichment Analysis.

    :param data: Gene expression data table, Pandas DataFrame, gct file.
    :param gene_sets: Enrichr Library name or .gmt gene sets file or dict of gene sets. Same input with GSEA.
    :param cls: A list or a .cls file format required for GSEA.
    :param str outdir: Results output directory.
    :param int permutation_num: Number of permutations for significance computation. Default: 1000.
    :param str permutation_type: Permutation type, "phenotype" for phenotypes, "gene_set" for genes.
    :param int min_size: Minimum allowed number of genes from gene set also the data set. Default: 15.
    :param int max_size: Maximum allowed number of genes from gene set also the data set. Default: 500.
    :param float weighted_score_type: Refer to :func:`algorithm.enrichment_score`. Default:1.
    :param method:  The method used to calculate a correlation or ranking. Default: 'log2_ratio_of_classes'.
                   Others methods are:

                   1. 'signal_to_noise'

                      You must have at least three samples for each phenotype to use this metric.
                      The larger the signal-to-noise ratio, the larger the differences of the means (scaled by the standard deviations);
                      that is, the more distinct the gene expression is in each phenotype and the more the gene acts as a “class marker.”

                   2. 't_test'

                      Uses the difference of means scaled by the standard deviation and number of samples.
                      Note: You must have at least three samples for each phenotype to use this metric.
                      The larger the tTest ratio, the more distinct the gene expression is in each phenotype
                      and the more the gene acts as a “class marker.”

                   3. 'ratio_of_classes' (also referred to as fold change).

                      Uses the ratio of class means to calculate fold change for natural scale data.

                   4. 'diff_of_classes'

                      Uses the difference of class means to calculate fold change for nature scale data

                   5. 'log2_ratio_of_classes'

                      Uses the log2 ratio of class means to calculate fold change for natural scale data.
                      This is the recommended statistic for calculating fold change for log scale data.

    :param bool ascending: Sorting order of rankings. Default: False.
    :param int processes: Number of Processes you are going to use. Default: 1.
    :param list figsize: Matplotlib figsize, accept a tuple or list, e.g. [width,height]. Default: [6.5,6].
    :param str format: Matplotlib figure format. Default: 'pdf'.
    :param int graph_num: Plot graphs for top sets of each phenotype.
    :param bool no_plot: If equals to True, no figure will be drawn. Default: False.
    :param seed: Random seed. expect an integer. Default:None.
    :param bool verbose: Bool, increase output verbosity, print out progress of your job, Default: False.

    :return: Return a GSEA obj. All results store to a dictionary, obj.results,
             where contains::

                 | {es: enrichment score,
                 |  nes: normalized enrichment score,
                 |  p: P-value,
                 |  fdr: FDR,
                 |  size: gene set size,
                 |  matched_size: genes matched to the data,
                 |  genes: gene names from the data set
                 |  ledge_genes: leading edge genes}

    """
    gs = GSEA(data, gene_sets, cls, outdir, min_size, max_size, permutation_num,
              weighted_score_type, permutation_type, method, ascending, processes,
               figsize, format, graph_num, no_plot, seed, verbose)
    gs.run()

    return gs

Response: Run GSEA analysis on a set of gene sets.


Instruction:
Explain what the following Python 3 code does
def ssgsea(data, gene_sets, outdir="ssGSEA_", sample_norm_method='rank', min_size=15, max_size=2000,
           permutation_num=0, weighted_score_type=0.25, scale=True, ascending=False, processes=1,
           figsize=(7,6), format='pdf', graph_num=20, no_plot=False, seed=None, verbose=False):
    """Run Gene Set Enrichment Analysis with single sample GSEA tool

    :param data: Expression table, pd.Series, pd.DataFrame, GCT file, or .rnk file format.
    :param gene_sets: Enrichr Library name or .gmt gene sets file or dict of gene sets. Same input with GSEA.
    :param outdir: Results output directory.
    :param str sample_norm_method: "Sample normalization method. Choose from {'rank', 'log', 'log_rank'}. Default: rank.

               1. 'rank': Rank your expression data, and transform by 10000*rank_dat/gene_numbers
               2. 'log' : Do not rank, but transform data by log(data + exp(1)), while data = data[data<1] =1.
               3. 'log_rank': Rank your expression data, and transform by log(10000*rank_dat/gene_numbers+ exp(1))
               4. 'custom': Do nothing, and use your own rank value to calculate enrichment score.

    see here: https://github.com/GSEA-MSigDB/ssGSEAProjection-gpmodule/blob/master/src/ssGSEAProjection.Library.R, line 86

    :param int min_size: Minimum allowed number of genes from gene set also the data set. Default: 15.
    :param int max_size: Maximum allowed number of genes from gene set also the data set. Default: 2000.
    :param int permutation_num: Number of permutations for significance computation. Default: 0.
    :param str weighted_score_type: Refer to :func:`algorithm.enrichment_score`. Default:0.25.
    :param bool scale: If True, normalize the scores by number of genes in the gene sets.
    :param bool ascending: Sorting order of rankings. Default: False.
    :param int processes: Number of Processes you are going to use. Default: 1.
    :param list figsize: Matplotlib figsize, accept a tuple or list, e.g. [width,height]. Default: [7,6].
    :param str format: Matplotlib figure format. Default: 'pdf'.
    :param int graph_num: Plot graphs for top sets of each phenotype.
    :param bool no_plot: If equals to True, no figure will be drawn. Default: False.
    :param seed: Random seed. expect an integer. Default:None.
    :param bool verbose: Bool, increase output verbosity, print out progress of your job, Default: False.

    :return: Return a ssGSEA obj.
             All results store to  a dictionary, access enrichment score by obj.resultsOnSamples,
             and normalized enrichment score by obj.res2d.
             if permutation_num > 0, additional results contain::

                 | {es: enrichment score,
                 |  nes: normalized enrichment score,
                 |  p: P-value,
                 |  fdr: FDR,
                 |  size: gene set size,
                 |  matched_size: genes matched to the data,
                 |  genes: gene names from the data set
                 |  ledge_genes: leading edge genes, if permutation_num >0}

    """

    ss = SingleSampleGSEA(data, gene_sets, outdir, sample_norm_method, min_size, max_size,
                          permutation_num, weighted_score_type, scale, ascending,
                          processes, figsize, format, graph_num, no_plot, seed, verbose)
    ss.run()
    return ss

Response: Run GSEA Enrichment Analysis with single sample GSEA


Instruction:
Can you implement a function in Python 3 that
runs GSEA Prerank analysis with pre - ranked correlation table.

Response:
def prerank(rnk, gene_sets, outdir='GSEA_Prerank', pheno_pos='Pos', pheno_neg='Neg',
            min_size=15, max_size=500, permutation_num=1000, weighted_score_type=1,
            ascending=False, processes=1, figsize=(6.5,6), format='pdf',
            graph_num=20, no_plot=False, seed=None, verbose=False):
    """ Run Gene Set Enrichment Analysis with pre-ranked correlation defined by user.

    :param rnk: pre-ranked correlation table or pandas DataFrame. Same input with ``GSEA`` .rnk file.
    :param gene_sets: Enrichr Library name or .gmt gene sets file or dict of gene sets. Same input with GSEA.
    :param outdir: results output directory.
    :param int permutation_num: Number of permutations for significance computation. Default: 1000.
    :param int min_size: Minimum allowed number of genes from gene set also the data set. Default: 15.
    :param int max_size: Maximum allowed number of genes from gene set also the data set. Defaults: 500.
    :param str weighted_score_type: Refer to :func:`algorithm.enrichment_score`. Default:1.
    :param bool ascending: Sorting order of rankings. Default: False.
    :param int processes: Number of Processes you are going to use. Default: 1.
    :param list figsize: Matplotlib figsize, accept a tuple or list, e.g. [width,height]. Default: [6.5,6].
    :param str format: Matplotlib figure format. Default: 'pdf'.
    :param int graph_num: Plot graphs for top sets of each phenotype.
    :param bool no_plot: If equals to True, no figure will be drawn. Default: False.
    :param seed: Random seed. expect an integer. Default:None.
    :param bool verbose: Bool, increase output verbosity, print out progress of your job, Default: False.

    :return: Return a Prerank obj. All results store to  a dictionary, obj.results,
             where contains::

                 | {es: enrichment score,
                 |  nes: normalized enrichment score,
                 |  p: P-value,
                 |  fdr: FDR,
                 |  size: gene set size,
                 |  matched_size: genes matched to the data,
                 |  genes: gene names from the data set
                 |  ledge_genes: leading edge genes}

    """
    pre = Prerank(rnk, gene_sets, outdir, pheno_pos, pheno_neg,
                  min_size, max_size, permutation_num, weighted_score_type,
                  ascending, processes, figsize, format, graph_num, no_plot, seed, verbose)
    pre.run()
    return pre


Instruction:
Write a Python 3 function that can
create temp directory and return filename

Response:
def prepare_outdir(self):
        """create temp directory."""
        self._outdir = self.outdir
        if self._outdir is None:
            self._tmpdir = TemporaryDirectory()
            self.outdir = self._tmpdir.name
        elif isinstance(self.outdir, str):
            mkdirs(self.outdir)
        else:
            raise Exception("Error parsing outdir: %s"%type(self.outdir))

        # handle gmt type
        if isinstance(self.gene_sets, str):
            _gset = os.path.split(self.gene_sets)[-1].lower().rstrip(".gmt")
        elif isinstance(self.gene_sets, dict):
            _gset = "blank_name"
        else:
            raise Exception("Error parsing gene_sets parameter for gene sets")

        logfile = os.path.join(self.outdir, "gseapy.%s.%s.log" % (self.module, _gset))
        return logfile


Instruction:
Create a Python 3 function for
setting cpu numbers to be used

Response:
def _set_cores(self):
        """set cpu numbers to be used"""

        cpu_num = cpu_count()-1
        if self._processes > cpu_num:
            cores = cpu_num
        elif self._processes < 1:
            cores = 1
        else:
            cores = self._processes
        # have to be int if user input is float
        self._processes = int(cores)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _load_ranking(self, rnk):
        # load data
        if isinstance(rnk, pd.DataFrame):
            rank_metric = rnk.copy()
            # handle dataframe with gene_name as index.
            if rnk.shape[1] == 1: rank_metric = rnk.reset_index()
        elif isinstance(rnk, pd.Series):
            rank_metric = rnk.reset_index()
        elif os.path.isfile(rnk):
            rank_metric = pd.read_csv(rnk, header=None, comment='#', sep="\t")
        else:
            raise Exception('Error parsing gene ranking values!')
        # sort ranking values from high to low
        rank_metric.sort_values(by=rank_metric.columns[1], ascending=self.ascending, inplace=True)
        # drop na values
        if rank_metric.isnull().any(axis=1).sum() >0:
            self._logger.warning("Input gene rankings contains NA values(gene name and ranking value), drop them all!")
            # print out NAs
            NAs = rank_metric[rank_metric.isnull().any(axis=1)]
            self._logger.debug('NAs list:\n'+NAs.to_string())
            rank_metric.dropna(how='any', inplace=True)
        # drop duplicate IDs, keep the first
        if rank_metric.duplicated(subset=rank_metric.columns[0]).sum() >0:
            self._logger.warning("Input gene rankings contains duplicated IDs, Only use the duplicated ID with highest value!")
            # print out duplicated IDs.
            dups = rank_metric[rank_metric.duplicated(subset=rank_metric.columns[0])]
            self._logger.debug('Dups list:\n'+dups.to_string())
            rank_metric.drop_duplicates(subset=rank_metric.columns[0], inplace=True, keep='first')
        # reset ranking index, because you have sort values and drop duplicates.
        rank_metric.reset_index(drop=True, inplace=True)
        rank_metric.columns = ['gene_name','rank']
        rankser = rank_metric.set_index('gene_name')['rank']
        self.ranking = rankser
        # return series
        return rankser

Response: Parse the ranking file and return a Pandas Series with the gene name indexed rankings and their corresponding IDs.


Instruction:
Write a Python 3 script to
load gene set dict from list of genes

Response:
def load_gmt(self, gene_list, gmt):
        """load gene set dict"""

        if isinstance(gmt, dict):
            genesets_dict = gmt
        elif isinstance(gmt, str):
            genesets_dict = self.parse_gmt(gmt)
        else:
            raise Exception("Error parsing gmt parameter for gene sets")

        subsets = list(genesets_dict.keys())
        self.n_genesets = len(subsets)
        for subset in subsets:
            subset_list = genesets_dict.get(subset)
            if isinstance(subset_list, set):
                subset_list = list(subset_list)
                genesets_dict[subset] = subset_list
            tag_indicator = np.in1d(gene_list, subset_list, assume_unique=True)
            tag_len = tag_indicator.sum()
            if  self.min_size <= tag_len <= self.max_size: continue
            del genesets_dict[subset]

        filsets_num = len(subsets) - len(genesets_dict)
        self._logger.info("%04d gene_sets have been filtered out when max_size=%s and min_size=%s"%(filsets_num, self.max_size, self.min_size))

        if filsets_num == len(subsets):
            self._logger.error("No gene sets passed through filtering condition!!!, try new parameters again!\n" +\
                               "Note: check gene name, gmt file format, or filtering size." )
            sys.exit(0)

        self._gmtdct=genesets_dict
        return genesets_dict


Instruction:
Can you tell what is the following Python 3 function doing
def parse_gmt(self, gmt):

        if gmt.lower().endswith(".gmt"):
            with open(gmt) as genesets:
                 genesets_dict = { line.strip().split("\t")[0]: line.strip().split("\t")[2:]
                                  for line in genesets.readlines()}
            return genesets_dict

        elif gmt in DEFAULT_LIBRARY:
            pass
        elif gmt in self.get_libraries():
            pass
        else:
            self._logger.error("No supported gene_sets: %s"%gmt)
            sys.exit(0)

        tmpname = "enrichr." + gmt + ".gmt"
        tempath = os.path.join(DEFAULT_CACHE_PATH, tmpname)
        # if file already download
        if os.path.isfile(tempath):
            self._logger.info("Enrichr library gene sets already downloaded in: %s, use local file"%DEFAULT_CACHE_PATH)
            return self.parse_gmt(tempath)
        else:
            return self._download_libraries(gmt)

Response: parse a gmt file and return a dict of all the genesets in the cache


Instruction:
Can you create a Python 3 function that
returns active enrichr library name. Offical API

Response:
def get_libraries(self, database=''):
        """return active enrichr library name.Offical API """

        lib_url='http://amp.pharm.mssm.edu/%sEnrichr/datasetStatistics'%database
        libs_json = json.loads(requests.get(lib_url).text)
        libs = [lib['libraryName'] for lib in libs_json['statistics']]
        return sorted(libs)


Instruction:
Can you generate the documentation for the following Python 3 function
def _download_libraries(self, libname):
        self._logger.info("Downloading and generating Enrichr library gene sets......")
        s = retry(5)
        # queery string
        ENRICHR_URL = 'http://amp.pharm.mssm.edu/Enrichr/geneSetLibrary'
        query_string = '?mode=text&libraryName=%s'
        # get
        response = s.get( ENRICHR_URL + query_string % libname, timeout=None)
        if not response.ok:
            raise Exception('Error fetching enrichment results, check internet connection first.')
        # reformat to dict and save to disk
        mkdirs(DEFAULT_CACHE_PATH)
        genesets_dict = {}
        outname = "enrichr.%s.gmt"%libname
        gmtout = open(os.path.join(DEFAULT_CACHE_PATH, outname), "w")
        for line in response.iter_lines(chunk_size=1024, decode_unicode='utf-8'):
            line=line.strip()
            k = line.split("\t")[0]
            v = list(map(lambda x: x.split(",")[0], line.split("\t")[2:]))
            genesets_dict.update({ k: v})
            outline = "%s\t\t%s\n"%(k, "\t".join(v))
            gmtout.write(outline)
        gmtout.close()

        return genesets_dict

Response: download enrichr libraries and generate a dict of gene sets


Instruction:
Make a summary of the following Python 3 code
def _plotting(self, rank_metric, results, graph_num, outdir,
                  format, figsize, pheno_pos='', pheno_neg=''):
        """ Plotting API.
            :param rank_metric: sorted pd.Series with rankings values.
            :param results: self.results
            :param data: preprocessed expression table

        """

        # no values need to be returned
        if self._outdir is None: return
        #Plotting
        top_term = self.res2d.index[:graph_num]
        # multi-threading
        pool = Pool(self._processes)
        for gs in top_term:
            hit = results.get(gs)['hits_indices']
            NES = 'nes' if self.module != 'ssgsea' else 'es'
            term = gs.replace('/','_').replace(":","_")
            outfile = '{0}/{1}.{2}.{3}'.format(self.outdir, term, self.module, self.format)
            # gseaplot(rank_metric=rank_metric, term=term, hits_indices=hit,
            #           nes=results.get(gs)[NES], pval=results.get(gs)['pval'],
            #           fdr=results.get(gs)['fdr'], RES=results.get(gs)['RES'],
            #           pheno_pos=pheno_pos, pheno_neg=pheno_neg, figsize=figsize,
            #           ofname=outfile)
            pool.apply_async(gseaplot, args=(rank_metric, term, hit, results.get(gs)[NES],
                                              results.get(gs)['pval'],results.get(gs)['fdr'],
                                              results.get(gs)['RES'],
                                              pheno_pos, pheno_neg,
                                              figsize, 'seismic', outfile))
            if self.module == 'gsea':
                outfile2 = "{0}/{1}.heatmap.{2}".format(self.outdir, term, self.format)
                # heatmap(df=self.heatmat.iloc[hit, :], title=term, ofname=outfile2,
                #         z_score=0, figsize=(self._width, len(hit)/2))
                pool.apply_async(heatmap, args=(self.heatmat.iloc[hit, :], 0, term,
                                               (self._width, len(hit)/2+2), 'RdBu_r',
                                                True, True, outfile2))
        pool.close()
        pool.join()

Response: Plots the rank_metric and data for the current object.


Instruction:
Can you implement a function in Python 3 that
reformats gsea results and save to txt

Response:
def _save_results(self, zipdata, outdir, module, gmt, rank_metric, permutation_type):
        """reformat gsea results, and save to txt"""

        res = OrderedDict()
        for gs, gseale, ind, RES in zipdata:
            rdict = OrderedDict()
            rdict['es'] = gseale[0]
            rdict['nes'] = gseale[1]
            rdict['pval'] = gseale[2]
            rdict['fdr'] = gseale[3]
            rdict['geneset_size'] = len(gmt[gs])
            rdict['matched_size'] = len(ind)
            #reformat gene list.
            _genes = rank_metric.index.values[ind]
            rdict['genes'] = ";".join([ str(g).strip() for g in _genes ])

            if self.module != 'ssgsea':
                # extract leading edge genes
                if rdict['es'] > 0:
                    # RES -> ndarray, ind -> list
                    idx = RES.argmax()
                    ldg_pos = list(filter(lambda x: x<= idx, ind))
                elif rdict['es'] < 0:
                    idx = RES.argmin()
                    ldg_pos = list(filter(lambda x: x >= idx, ind))
                else:
                    ldg_pos = ind # es == 0 ?
                rdict['ledge_genes'] = ';'.join(list(map(str,rank_metric.iloc[ldg_pos].index)))

            rdict['RES'] = RES
            rdict['hits_indices'] = ind
            # save to one odict
            res[gs] = rdict
        # save
        self.results  = res
        # save to dataframe
        res_df = pd.DataFrame.from_dict(res, orient='index')
        res_df.index.name = 'Term'
        res_df.drop(['RES','hits_indices'], axis=1, inplace=True)
        res_df.sort_values(by=['fdr','pval'], inplace=True)
        self.res2d = res_df

        if self._outdir is None: return
        out = os.path.join(outdir,'gseapy.{b}.{c}.report.csv'.format(b=module, c=permutation_type))
        if self.module == 'ssgsea':
            out = out.replace(".csv",".txt")
            with open(out, 'a') as f:
                f.write('# normalize enrichment scores by random permutation procedure (GSEA method)\n')
                f.write("# might not proper for publication\n")
                res_df.to_csv(f, sep='\t')
        else:
            res_df.to_csv(out)

        return


Instruction:
Implement a function in Python 3 to
load the data from the file and return a DataFrame with the data as a DataFrame.

Response:
def load_data(self, cls_vec):
        """pre-processed the data frame.new filtering methods will be implement here.
        """
        # read data in
        if isinstance(self.data, pd.DataFrame) :
            exprs = self.data.copy()
            # handle index is gene_names
            if exprs.index.dtype == 'O':
                exprs = exprs.reset_index()
        elif os.path.isfile(self.data) :
            # GCT input format?
            if self.data.endswith("gct"):
                exprs = pd.read_csv(self.data, skiprows=1, comment='#',sep="\t")
            else:
                exprs = pd.read_csv(self.data, comment='#',sep="\t")
        else:
            raise Exception('Error parsing gene expression DataFrame!')

        #drop duplicated gene names
        if exprs.iloc[:,0].duplicated().sum() > 0:
            self._logger.warning("Warning: dropping duplicated gene names, only keep the first values")
            exprs.drop_duplicates(subset=exprs.columns[0], inplace=True) #drop duplicate gene_names.
        if exprs.isnull().any().sum() > 0:
            self._logger.warning("Warning: Input data contains NA, filled NA with 0")
            exprs.dropna(how='all', inplace=True) #drop rows with all NAs
            exprs = exprs.fillna(0)
        # set gene name as index
        exprs.set_index(keys=exprs.columns[0], inplace=True)
        # select numberic columns
        df = exprs.select_dtypes(include=[np.number])
        # drop any genes which std ==0
        df_std =  df.groupby(by=cls_vec, axis=1).std()
        df =  df[~df_std.isin([0]).any(axis=1)]
        df = df + 0.00001 # we don't like zeros!!!
        return df


Instruction:
Can you tell what is the following Python 3 function doing
def run(self):

        assert self.permutation_type in ["phenotype", "gene_set"]
        assert self.min_size <= self.max_size

        # Start Analysis
        self._logger.info("Parsing data files for GSEA.............................")
        # phenotype labels parsing
        phenoPos, phenoNeg, cls_vector = gsea_cls_parser(self.classes)
        # select correct expression genes and values.
        dat = self.load_data(cls_vector)
        # data frame must have length > 1
        assert len(dat) > 1
        # ranking metrics calculation.
        dat2 = ranking_metric(df=dat, method=self.method, pos=phenoPos, neg=phenoNeg,
                              classes=cls_vector, ascending=self.ascending)
        self.ranking = dat2
        # filtering out gene sets and build gene sets dictionary
        gmt = self.load_gmt(gene_list=dat2.index.values, gmt=self.gene_sets)

        self._logger.info("%04d gene_sets used for further statistical testing....."% len(gmt))
        self._logger.info("Start to run GSEA...Might take a while..................")
        # cpu numbers
        self._set_cores()
        # compute ES, NES, pval, FDR, RES
        dataset = dat if self.permutation_type =='phenotype' else dat2
        gsea_results,hit_ind,rank_ES, subsets = gsea_compute_tensor(data=dataset, gmt=gmt, n=self.permutation_num,
                                                             weighted_score_type=self.weighted_score_type,
                                                             permutation_type=self.permutation_type,
                                                             method=self.method,
                                                             pheno_pos=phenoPos, pheno_neg=phenoNeg,
                                                             classes=cls_vector, ascending=self.ascending,
                                                             processes=self._processes, seed=self.seed)

        self._logger.info("Start to generate GSEApy reports and figures............")
        res_zip = zip(subsets, list(gsea_results), hit_ind, rank_ES)
        self._save_results(zipdata=res_zip, outdir=self.outdir, module=self.module,
                                   gmt=gmt, rank_metric=dat2, permutation_type=self.permutation_type)

        # reorder datarame for heatmap
        self._heatmat(df=dat.loc[dat2.index], classes=cls_vector,
                      pheno_pos=phenoPos, pheno_neg=phenoNeg)
        # Plotting
        if not self._noplot:
            self._plotting(rank_metric=dat2, results=self.results,
                           graph_num=self.graph_num, outdir=self.outdir,
                           figsize=self.figsize, format=self.format,
                           pheno_pos=phenoPos, pheno_neg=phenoNeg)

        self._logger.info("Congratulations. GSEApy ran successfully.................\n")
        if self._outdir is None:
            self._tmpdir.cleanup()

        return

Response: This method is called by the base class to run the GSEA analysis.


Instruction:
How would you explain what the following Python 3 function does
def run(self):

        assert self.min_size <= self.max_size

        # parsing rankings
        dat2 = self._load_ranking(self.rnk)
        assert len(dat2) > 1

        # cpu numbers
        self._set_cores()
        # Start Analysis
        self._logger.info("Parsing data files for GSEA.............................")
        # filtering out gene sets and build gene sets dictionary
        gmt = self.load_gmt(gene_list=dat2.index.values, gmt=self.gene_sets)

        self._logger.info("%04d gene_sets used for further statistical testing....."% len(gmt))
        self._logger.info("Start to run GSEA...Might take a while..................")
        # compute ES, NES, pval, FDR, RES
        gsea_results, hit_ind,rank_ES, subsets = gsea_compute(data=dat2, n=self.permutation_num, gmt=gmt,
                                                              weighted_score_type=self.weighted_score_type,
                                                              permutation_type='gene_set', method=None,
                                                              pheno_pos=self.pheno_pos, pheno_neg=self.pheno_neg,
                                                              classes=None, ascending=self.ascending,
                                                              processes=self._processes, seed=self.seed)
        self._logger.info("Start to generate gseapy reports, and produce figures...")
        res_zip = zip(subsets, list(gsea_results), hit_ind, rank_ES)
        self._save_results(zipdata=res_zip, outdir=self.outdir, module=self.module,
                                   gmt=gmt, rank_metric=dat2, permutation_type="gene_sets")

        # Plotting
        if not self._noplot:
            self._plotting(rank_metric=dat2, results=self.results,
                           graph_num=self.graph_num, outdir=self.outdir,
                           figsize=self.figsize, format=self.format,
                           pheno_pos=self.pheno_pos, pheno_neg=self.pheno_neg)

        self._logger.info("Congratulations. GSEApy runs successfully................\n")
        if self._outdir is None:
            self._tmpdir.cleanup()

        return

Response: Run the GSEA prerank workflow.


Instruction:
Create a Python 3 function for
normalizing the samples according to the sample normalization method

Response:
def norm_samples(self, dat):
        """normalization samples
           see here: http://rowley.mit.edu/caw_web/ssGSEAProjection/ssGSEAProjection.Library.R
        """

        if self.sample_norm_method == 'rank':
            data = dat.rank(axis=0, method='average', na_option='bottom')
            data = 10000*data / data.shape[0]
        elif self.sample_norm_method == 'log_rank':
            data = dat.rank(axis=0, method='average', na_option='bottom')
            data = log(10000*data / data.shape[0] + exp(1))
        elif self.sample_norm_method == 'log':
            dat[dat < 1] = 1
            data = log(dat + exp(1))
        elif self.sample_norm_method == 'custom':
            self._logger.info("Use custom rank metric for ssGSEA")
            data = dat
        else:
            sys.stderr.write("No supported method: %s"%self.sample_norm_method)
            sys.exit(0)

        return data


Instruction:
Can you generate a brief explanation for the following Python 3 code
def run(self):
        self._logger.info("Parsing data files for ssGSEA...........................")
        # load data
        data = self.load_data()
        # normalized samples, and rank
        normdat = self.norm_samples(data)
        # filtering out gene sets and build gene sets dictionary
        gmt = self.load_gmt(gene_list=normdat.index.values, gmt=self.gene_sets)
        self._logger.info("%04d gene_sets used for further statistical testing....."% len(gmt))
        # set cpu numbers
        self._set_cores()
        # start analsis
        self._logger.info("Start to run ssGSEA...Might take a while................")
        if self.permutation_num == 0 :
            # ssGSEA without permutation
            self.runSamples(df=normdat, gmt=gmt)
        else:
            # run permutation procedure and calculate pvals, fdrs
            self._logger.warning("run ssGSEA with permutation procedure, don't use these part of results for publication.")
            self.runSamplesPermu(df=normdat, gmt=gmt)
        # clean up all outputs if _outdir is None
        if self._outdir is None:
            self._tmpdir.cleanup()

Response: run entry and return a dictionary of the most recent analisymmetric state.


Instruction:
How would you explain what the following Python 3 function does
def runSamplesPermu(self, df, gmt=None):

        assert self.min_size <= self.max_size
        mkdirs(self.outdir)
        self.resultsOnSamples = OrderedDict()
        outdir = self.outdir
        # iter throught each sample
        for name, ser in df.iteritems():
            self.outdir = os.path.join(outdir, str(name))
            self._logger.info("Run Sample: %s " % name)
            mkdirs(self.outdir)
            # sort ranking values from high to low or reverse
            dat2 = ser.sort_values(ascending=self.ascending)
            # reset integer index, or caused unwanted problems
            # df.reset_index(drop=True, inplace=True)

            # compute ES, NES, pval, FDR, RES
            gsea_results, hit_ind,rank_ES, subsets = gsea_compute(data=dat2, n=self.permutation_num, gmt=gmt,
                                                                  weighted_score_type=self.weighted_score_type,
                                                                  permutation_type='gene_set', method=None,
                                                                  pheno_pos='', pheno_neg='',
                                                                  classes=None, ascending=self.ascending,
                                                                  processes=self._processes,
                                                                  seed=self.seed, single=True, scale=self.scale)

            # write file
            res_zip = zip(subsets, list(gsea_results), hit_ind, rank_ES)
            self._save_results(zipdata=res_zip, outdir=self.outdir, module=self.module,
                                       gmt=gmt, rank_metric=dat2, permutation_type="gene_sets")
            self.resultsOnSamples[name] = self.res2d.es
            # plotting
            if self._noplot: continue
            self._logger.info("Plotting Sample: %s \n" % name)
            self._plotting(rank_metric=dat2, results=self.results,
                           graph_num=self.graph_num, outdir=self.outdir,
                           figsize=self.figsize, format=self.format)

        # save es, nes to file
        self._save(outdir)

        return

Response: Run the GSEA algorithm with a single sample with a permutation procedure.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def runSamples(self, df, gmt=None):

        # df.index.values are gene_names
        # Save each sample results to odict
        self.resultsOnSamples = OrderedDict()
        outdir = self.outdir
        # run ssgsea for gct expression matrix
        #multi-threading
        subsets = sorted(gmt.keys())
        tempes=[]
        names=[]
        rankings=[]
        pool = Pool(processes=self._processes)
        for name, ser in df.iteritems():
            #prepare input
            dat = ser.sort_values(ascending=self.ascending)
            rankings.append(dat)
            names.append(name)
            genes_sorted, cor_vec = dat.index.values, dat.values
            rs = np.random.RandomState(self.seed)
            # apply_async
            tempes.append(pool.apply_async(enrichment_score_tensor,
                                           args=(genes_sorted, cor_vec, gmt,
                                               self.weighted_score_type,
                                               self.permutation_num, rs, True,
                                               self.scale)))
        pool.close()
        pool.join()
        # save results and plotting
        for i, temp in enumerate(tempes):
            name, rnk = names[i], rankings[i]
            self._logger.info("Calculate Enrichment Score for Sample: %s "%name)
            es, esnull, hit_ind, RES = temp.get()
            # create results subdir
            self.outdir= os.path.join(outdir, str(name))
            mkdirs(self.outdir)
            # save results
            self.resultsOnSamples[name] = pd.Series(data=es, index=subsets, name=name)
            # plotting
            if self._noplot: continue
            self._logger.info("Plotting Sample: %s \n" % name)
            for i, term in enumerate(subsets):
                term = term.replace('/','_').replace(":","_")
                outfile = '{0}/{1}.{2}.{3}'.format(self.outdir, term, self.module, self.format)
                gseaplot(rank_metric=rnk, term=term,
                         hits_indices=hit_ind[i], nes=es[i], pval=1, fdr=1,
                         RES=RES[i], pheno_pos='', pheno_neg='',
                         figsize=self.figsize, ofname=outfile)
        # save es, nes to file
        self._save(outdir)

        return

Response: Run multi - threading GSEA for each sample.


Instruction:
How would you explain what the following Python 3 function does
def _save(self, outdir):
        # save raw ES to one csv file
        samplesRawES = pd.DataFrame(self.resultsOnSamples)
        samplesRawES.index.name = 'Term|ES'
        # normalize enrichment scores by using the entire data set, as indicated
        # by Barbie et al., 2009, online methods, pg. 2
        samplesNES = samplesRawES / (samplesRawES.values.max() - samplesRawES.values.min())
        samplesNES = samplesNES.copy()
        samplesNES.index.rename('Term|NES', inplace=True)
        self.res2d = samplesNES
        self._logger.info("Congratulations. GSEApy runs successfully................\n")
        if self._outdir is None: return
        # write es
        outESfile = os.path.join(outdir, "gseapy.samples.raw.es.txt")
        with open(outESfile, 'a') as f:
            if self.scale:
                f.write('# scale the enrichment scores by number of genes in the gene sets\n')
                f.write('# this normalization has not effects on the final NES, ' + \
                        'as indicated by Barbie et al., 2009, online methods, pg. 2\n')
            else:
                f.write('# raw enrichment scores of all data\n')
                f.write('# no scale es by numbers of genes in the gene sets\n')
            samplesRawES.to_csv(f, sep='\t')

        outNESfile = os.path.join(outdir, "gseapy.samples.normalized.es.txt")
        with open(outNESfile, 'a') as f:
            f.write('# normalize enrichment scores by using the entire data set\n')
            f.write('# as indicated by Barbie et al., 2009, online methods, pg. 2\n')
            samplesNES.to_csv(f, sep='\t')
        return

Response: save raw es and stats


Instruction:
Can you tell what is the following Python 3 function doing
def run(self):
        assert self.min_size <= self.max_size
        assert self.fignum > 0
        import glob
        from bs4 import BeautifulSoup

        # parsing files.......
        try:
            results_path = glob.glob(self.indir+'*/edb/results.edb')[0]
            rank_path =  glob.glob(self.indir+'*/edb/*.rnk')[0]
            gene_set_path =  glob.glob(self.indir+'*/edb/gene_sets.gmt')[0]
        except IndexError as e:
            sys.stderr.write("Could not locate GSEA files in the given directory!")
            sys.exit(1)
        # extract sample names from .cls file
        cls_path = glob.glob(self.indir+'*/edb/*.cls')
        if cls_path:
            pos, neg, classes = gsea_cls_parser(cls_path[0])
        else:
            # logic for prerank results
            pos, neg = '',''
        # start reploting
        self.gene_sets=gene_set_path
        # obtain gene sets
        gene_set_dict = self.parse_gmt(gmt=gene_set_path)
        # obtain rank_metrics
        rank_metric = self._load_ranking(rank_path)
        correl_vector =  rank_metric.values
        gene_list = rank_metric.index.values
        # extract each enriment term in the results.edb files and plot.
        database = BeautifulSoup(open(results_path), features='xml')
        length = len(database.findAll('DTG'))
        fig_num = self.fignum if self.fignum <= length else length
        for idx in range(fig_num):
            # extract statistical resutls from results.edb file
            enrich_term, hit_ind, nes, pval, fdr= gsea_edb_parser(results_path, index=idx)
            gene_set = gene_set_dict.get(enrich_term)
            # calculate enrichment score
            RES = enrichment_score(gene_list=gene_list,
                                   correl_vector=correl_vector,
                                   gene_set=gene_set,
                                   weighted_score_type=self.weighted_score_type,
                                   nperm=0)[-1]
            # plotting
            term = enrich_term.replace('/','_').replace(":","_")
            outfile = '{0}/{1}.{2}.{3}'.format(self.outdir, term, self.module, self.format)
            gseaplot(rank_metric=rank_metric, term=enrich_term,
                         hits_indices=hit_ind, nes=nes, pval=pval, fdr=fdr,
                         RES=RES, pheno_pos=pos, pheno_neg=neg,
                         figsize=self.figsize, ofname=outfile)

        self._logger.info("Congratulations! Your plots have been reproduced successfully!\n")

Response: main function that parses the results. edb files and plots the GSEA data.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def prepare_outdir(self):
        self._outdir = self.outdir
        if self._outdir is None:
            self._tmpdir = TemporaryDirectory()
            self.outdir = self._tmpdir.name
        elif isinstance(self.outdir, str):
            mkdirs(self.outdir)
        else:
            raise Exception("Error parsing outdir: %s"%type(self.outdir))

        # handle gene_sets
        logfile = os.path.join(self.outdir, "gseapy.%s.%s.log" % (self.module, self.descriptions))
        return logfile

Response: create temp directory and return logfile.


Instruction:
Write a Python 3 script for
parsing gene_sets input file type

Response:
def parse_genesets(self):
        """parse gene_sets input file type"""

        enrichr_library = self.get_libraries()
        if isinstance(self.gene_sets, list):
            gss = self.gene_sets
        elif isinstance(self.gene_sets, str):
            gss = [ g.strip() for g in self.gene_sets.strip().split(",") ]
        elif isinstance(self.gene_sets, dict):
            gss = [self.gene_sets]
        else:
            raise Exception("Error parsing enrichr libraries, please provided corrected one")

        # gss: a list contain .gmt, dict, enrichr_liraries.
        # now, convert .gmt to dict
        gss_exist = []
        for g in gss:
            if isinstance(g, dict):
                gss_exist.append(g)
                continue

            if isinstance(g, str):
                if g in enrichr_library:
                    gss_exist.append(g)
                    continue
                if g.lower().endswith(".gmt") and os.path.exists(g):
                    self._logger.info("User Defined gene sets is given: %s"%g)
                    with open(g) as genesets:
                        g_dict = { line.strip().split("\t")[0]: line.strip().split("\t")[2:]
                                        for line in genesets.readlines() }
                    gss_exist.append(g_dict)
        return gss_exist


Instruction:
Explain what the following Python 3 code does
def send_genes(self, gene_list, url):
        payload = {
          'list': (None, gene_list),
          'description': (None, self.descriptions)
           }
        # response
        response = requests.post(url, files=payload)
        if not response.ok:
            raise Exception('Error analyzing gene list')
        sleep(1)
        job_id = json.loads(response.text)

        return job_id

Response: send gene list to enrichr server


Instruction:
Write a Python 3 script for
comparing the genes sent and received to get successfully recognized genes

Response:
def check_genes(self, gene_list, usr_list_id):
        '''
        Compare the genes sent and received to get successfully recognized genes
        '''
        response = requests.get('http://amp.pharm.mssm.edu/Enrichr/view?userListId=%s' % usr_list_id)
        if not response.ok:
            raise Exception('Error getting gene list back')
        returnedL = json.loads(response.text)["genes"]
        returnedN = sum([1 for gene in gene_list if gene in returnedL])
        self._logger.info('{} genes successfully recognized by Enrichr'.format(returnedN))


Instruction:
Given the following Python 3 function, write the documentation
def get_results(self, gene_list):
        ADDLIST_URL = 'http://amp.pharm.mssm.edu/%sEnrichr/addList'%self._organism
        job_id = self.send_genes(gene_list, ADDLIST_URL)
        user_list_id = job_id['userListId']

        RESULTS_URL = 'http://amp.pharm.mssm.edu/%sEnrichr/export'%self._organism
        query_string = '?userListId=%s&filename=%s&backgroundType=%s'
        # set max retries num =5
        s = retry(num=5)
        filename = "%s.%s.reports" % (self._gs, self.descriptions)
        url = RESULTS_URL + query_string % (user_list_id, filename, self._gs)
        response = s.get(url, stream=True, timeout=None)
        # response = requests.get(RESULTS_URL + query_string % (user_list_id, gene_set))
        sleep(1)
        res = pd.read_csv(StringIO(response.content.decode('utf-8')),sep="\t")
        return [job_id['shortId'], res]

Response: Get the results from the enrichr API


Instruction:
Can you implement a function in Python 3 that
selects Enrichr organism from below

Response:
def get_organism(self):
        """Select Enrichr organism from below:

           Human & Mouse: H. sapiens & M. musculus
           Fly: D. melanogaster
           Yeast: S. cerevisiae
           Worm: C. elegans
           Fish: D. rerio

        """

        organism = {'default': ['', 'hs', 'mm', 'human','mouse',
                                'homo sapiens', 'mus musculus',
                                'h. sapiens', 'm. musculus'],
                    'Fly': ['fly', 'd. melanogaster', 'drosophila melanogaster'],
                    'Yeast': ['yeast', 's. cerevisiae', 'saccharomyces cerevisiae'],
                    'Worm': ['worm', 'c. elegans', 'caenorhabditis elegans', 'nematode'],
                    'Fish': ['fish', 'd. rerio', 'danio rerio', 'zebrafish']
                 }

        for k, v in organism.items():
            if self.organism.lower() in v :
                self._organism = k

        if self._organism is None:
            raise Exception("No supported organism found !!!")

        if self._organism == 'default':
            self._organism = ''
        return


Instruction:
Write a Python 3 script to
enrich the object with the current state of the object

Response:
def enrich(self, gmt):
        """use local mode

        p = p-value computed using the Fisher exact test (Hypergeometric test)

        Not implemented here:

            combine score = log(p)·z

        see here: http://amp.pharm.mssm.edu/Enrichr/help#background&q=4

        columns contain:

            Term Overlap P-value Adjusted_P-value Genes

        """
        if isscalar(self.background):
            if isinstance(self.background, int) or self.background.isdigit():
                self._bg = int(self.background)
            elif isinstance(self.background, str):
                # self.background = set(reduce(lambda x,y: x+y, gmt.values(),[]))
                self._bg = self.get_background()
                self._logger.info("Background: found %s genes"%(len(self._bg)))
            else:
                raise Exception("Unsupported background data type")
        else:
            # handle array object: nd.array, list, tuple, set, Series
            try:
                it = iter(self.background)
                self._bg = set(self.background)
            except TypeError:
                self._logger.error("Unsupported background data type")
        # statistical testing
        hgtest = list(calc_pvalues(query=self._gls, gene_sets=gmt,
                                   background=self._bg))
        if len(hgtest) > 0:
            terms, pvals, olsz, gsetsz, genes = hgtest
            fdrs, rej = multiple_testing_correction(ps = pvals,
                                                    alpha=self.cutoff,
                                                    method='benjamini-hochberg')
            # save to a dataframe
            odict = OrderedDict()
            odict['Term'] = terms
            odict['Overlap'] = list(map(lambda h,g: "%s/%s"%(h, g), olsz, gsetsz))
            odict['P-value'] = pvals
            odict['Adjusted P-value'] = fdrs
            # odict['Reject (FDR< %s)'%self.cutoff ] = rej
            odict['Genes'] = [";".join(g) for g in genes]
            res = pd.DataFrame(odict)
            return res
        return


Instruction:
Make a summary of the following Python 3 code
def run(self):

        # set organism
        self.get_organism()
        # read input file
        genes_list = self.parse_genelists()
        gss = self.parse_genesets()
        # if gmt
        self._logger.info("Connecting to Enrichr Server to get latest library names")
        if len(gss) < 1:
            sys.stderr.write("Not validated Enrichr library name provided\n")
            sys.stdout.write("Hint: use get_library_name() to view full list of supported names")
            sys.exit(1)
        self.results = pd.DataFrame()

        for g in gss:
            if isinstance(g, dict):
                ## local mode
                res = self.enrich(g)
                shortID, self._gs = str(id(g)), "CUSTOM%s"%id(g)
                if res is None:
                    self._logger.info("No hits return, for gene set: Custom%s"%shortID)
                    continue
            else:
                ## online mode
                self._gs = str(g)
                self._logger.debug("Start Enrichr using library: %s" % (self._gs))
                self._logger.info('Analysis name: %s, Enrichr Library: %s' % (self.descriptions, self._gs))
                shortID, res = self.get_results(genes_list)
                # Remember gene set library used
            res.insert(0, "Gene_set", self._gs)
            # Append to master dataframe
            self.results = self.results.append(res, ignore_index=True, sort=True)
            self.res2d = res
            if self._outdir is None: continue
            self._logger.info('Save file of enrichment results: Job Id:' + str(shortID))
            outfile = "%s/%s.%s.%s.reports.txt" % (self.outdir, self._gs, self.descriptions, self.module)
            self.res2d.to_csv(outfile, index=False, encoding='utf-8', sep="\t")
            # plotting
            if not self.__no_plot:
                msg = barplot(df=res, cutoff=self.cutoff, figsize=self.figsize,
                              top_term=self.__top_term, color='salmon',
                              title=self._gs,
                              ofname=outfile.replace("txt", self.format))
                if msg is not None : self._logger.warning(msg)
            self._logger.info('Done.\n')
        # clean up tmpdir
        if self._outdir is None: self._tmpdir.cleanup()

        return

Response: run enrichr for one sample gene list but multi - libraries