Instruction:
Can you generate the documentation for the following Python 3 function
def build_case(case, vcf_individuals=None, case_id=None, vcf_path=None, sv_individuals=None,
               vcf_sv_path=None, nr_variants=None, nr_sv_variants=None, profiles=None,
               matches=None, profile_path=None):
    """Build a Case from the given information

    Args:
        case(ped_parser.Family): A family object
        vcf_individuals(list): Show the order of inds in vcf file
        case_id(str): If another name than the one in family file should be used
        vcf_path(str)
        sv_individuals(list): Show the order of inds in vcf file
        vcf_sv_path(str)
        nr_variants(int)
        nr_sv_variants(int)
        profiles(dict): The profiles for each sample in vcf
        matches(dict(list)): list of similar samples for each sample in vcf.

    Returns:
        case_obj(models.Case)
    """
    # Create a dict that maps the ind ids to the position they have in vcf
    individual_positions = get_individual_positions(vcf_individuals)
    sv_individual_positions = get_individual_positions(sv_individuals)

    family_id = None
    if case:
        if not case.affected_individuals:
            LOG.warning("No affected individuals could be found in ped file")
        family_id = case.family_id

    # If case id is given manually we use that one
    case_id = case_id or family_id
    if case_id is None:
        raise CaseError

    case_obj = Case(
        case_id=case_id,
    )

    if vcf_path:
        case_obj['vcf_path'] = vcf_path
        case_obj['nr_variants'] = nr_variants

    if vcf_sv_path:
        case_obj['vcf_sv_path'] = vcf_sv_path
        case_obj['nr_sv_variants'] = nr_sv_variants

    if profile_path:
        case_obj['profile_path'] = profile_path

    ind_objs = []
    if case:
        if individual_positions:
            _ind_pos = individual_positions
        else:
            _ind_pos = sv_individual_positions

        for ind_id in case.individuals:
            individual = case.individuals[ind_id]
            try:
                #If a profile dict exists, get the profile for ind_id
                profile = profiles[ind_id] if profiles else None
                #If matching samples are found, get these samples for ind_id
                similar_samples = matches[ind_id] if matches else None
                ind_obj = Individual(
                    ind_id=ind_id,
                    case_id=case_id,
                    ind_index=_ind_pos[ind_id],
                    sex=individual.sex,
                    profile=profile,
                    similar_samples=similar_samples
                )
                ind_objs.append(dict(ind_obj))
            except KeyError:
                raise CaseError("Ind %s in ped file does not exist in VCF", ind_id)
    else:
        # If there where no family file we can create individuals from what we know
        for ind_id in individual_positions:

            profile = profiles[ind_id] if profiles else None
            similar_samples = matches[ind_id] if matches else None
            ind_obj = Individual(
                ind_id = ind_id,
                case_id = case_id,
                ind_index=individual_positions[ind_id],
                profile=profile,
                similar_samples=similar_samples
            )
            ind_objs.append(dict(ind_obj))

    # Add individuals to the correct variant type
    for ind_obj in ind_objs:
        if vcf_sv_path:
            case_obj['sv_individuals'].append(dict(ind_obj))
            case_obj['_sv_inds'][ind_obj['ind_id']] = dict(ind_obj)
        if vcf_path:
            case_obj['individuals'].append(dict(ind_obj))
            case_obj['_inds'][ind_obj['ind_id']] = dict(ind_obj)

    return case_obj

Response: Build a Case object from the given information.


Instruction:
Given the following Python 3 function, write the documentation
def generate_poisson_data(centers, n_cells, cluster_probs=None):
    genes, clusters = centers.shape
    output = np.zeros((genes, n_cells))
    if cluster_probs is None:
        cluster_probs = np.ones(clusters)/clusters
    labels = []
    for i in range(n_cells):
        c = np.random.choice(range(clusters), p=cluster_probs)
        labels.append(c)
        output[:,i] = np.random.poisson(centers[:,c])
    return output, np.array(labels)

Response: Generates poisson - distributed data given a set of means for each cluster.


Instruction:
Can you generate the documentation for the following Python 3 function
def generate_zip_data(M, L, n_cells, cluster_probs=None):
    genes, clusters = M.shape
    output = np.zeros((genes, n_cells))
    if cluster_probs is None:
        cluster_probs = np.ones(clusters)/clusters
    zip_p = np.random.random((genes, n_cells))
    labels = []
    for i in range(n_cells):
        c = np.random.choice(range(clusters), p=cluster_probs)
        labels.append(c)
        output[:,i] = np.where(zip_p[:,i] < L[:,c], 0, np.random.poisson(M[:,c]))
    return output, np.array(labels)

Response: Generates zero - inflated poisson - distributed data given a set of means and zero probs for each cluster.


Instruction:
Given the following Python 3 function, write the documentation
def generate_state_data(means, weights):
    x_true = np.dot(means, weights)
    sample = np.random.poisson(x_true)
    return sample.astype(float)

Response: Generates data according to the Poisson Convex Mixture Model.


Instruction:
Create a Python 3 function to
generate data according to Zero - inflated Poisson Convex Mixture Model.

Response:
def generate_zip_state_data(means, weights, z):
    """
    Generates data according to the Zero-inflated Poisson Convex Mixture Model.

    Args:
        means (array): Cell types- genes x clusters
        weights (array): Cell cluster assignments- clusters x cells
        z (float): zero-inflation parameter

    Returns:
        data matrix - genes x cells
    """
    x_true = np.dot(means, weights)
    sample = np.random.poisson(x_true)
    random = np.random.random(x_true.shape)
    x_true[random < z] = 0
    return sample.astype(float)


Instruction:
Can you create a Python 3 function that
generates data according to the Negative Binomial Convex Mixture Model.

Response:
def generate_nb_state_data(means, weights, R):
    """
    Generates data according to the Negative Binomial Convex Mixture Model.

    Args:
        means (array): Cell types- genes x clusters
        weights (array): Cell cluster assignments- clusters x cells
        R (array): dispersion parameter - 1 x genes

    Returns:
        data matrix - genes x cells
    """
    cells = weights.shape[1]
    # x_true = true means
    x_true = np.dot(means, weights)
    # convert means into P
    R_ = np.tile(R, (cells, 1)).T
    P_true = x_true/(R_ + x_true)
    sample = np.random.negative_binomial(np.tile(R, (cells, 1)).T, P_true)
    return sample.astype(float)


Instruction:
Explain what the following Python 3 code does
def generate_nb_states(n_states, n_cells, n_genes):
    W = np.random.dirichlet([1]*n_states, size=(n_cells,))
    W = W.T
    M = np.random.random((n_genes, n_states))*100
    R = np.random.randint(1, 100, n_genes)
    return M, W, R

Response: Generates the means and weights for the Negative Binomial Mixture Model.


Instruction:
How would you explain what the following Python 3 function does
def generate_poisson_states(n_states, n_cells, n_genes):
    W = np.random.dirichlet([1]*n_states, size=(n_cells,))
    W = W.T
    M = np.random.random((n_genes, n_states))*100
    return M, W

Response: Generates the Poisson states for a given set of genes and number of cells.


Instruction:
Here you have a function in Python 3, explain what it does
def generate_poisson_lineage(n_states, n_cells_per_cluster, n_genes, means=300):
    # means...
    M = np.random.random((n_genes, n_states))*means
    center = M.mean(1)
    W = np.zeros((n_states, n_cells_per_cluster*n_states))
    # TODO
    # start at a center where all the clusters have equal probability, and for
    # each cluster, interpolate linearly towards the cluster.
    index = 0
    means = np.array([1.0/n_states]*n_states)
    for c in range(n_states):
        for i in range(n_cells_per_cluster):
            w = np.copy(means)
            new_value = w[c] + i*(1.0 - 1.0/n_states)/n_cells_per_cluster
            w[:] = (1.0 - new_value)/(n_states - 1.0)
            w[c] = new_value
            W[:, index] = w
            index += 1
    return M, W

Response: Generates a lineage for each state.


Instruction:
Implement a Python 3 function for
generating negative binomial data for a set of cells in a cluster.

Response:
def generate_nb_data(P, R, n_cells, assignments=None):
    """
    Generates negative binomial data

    Args:
        P (array): genes x clusters
        R (array): genes x clusters
        n_cells (int): number of cells
        assignments (list): cluster assignment of each cell. Default:
            random uniform

    Returns:
        data array with shape genes x cells
        labels - array of cluster labels
    """
    genes, clusters = P.shape
    output = np.zeros((genes, n_cells))
    if assignments is None:
        cluster_probs = np.ones(clusters)/clusters
    labels = []
    for i in range(n_cells):
        if assignments is None:
            c = np.random.choice(range(clusters), p=cluster_probs)
        else:
            c = assignments[i]
        labels.append(c)
        # because numpy's negative binomial, r is the number of successes
        output[:,i] = np.random.negative_binomial(R[:,c], 1.0-P[:,c])
    return output, np.array(labels)


Instruction:
Create a Python 3 function for
visualizing a Poisson W.

Response:
def visualize_poisson_w(w, labels, filename, method='pca', figsize=(18,10), title='', **scatter_options):
    """
    Saves a scatter plot of a visualization of W, the result from Poisson SE.
    """
    if method == 'pca':
        pca = PCA(2)
        r_dim_red = pca.fit_transform(w.T).T
    elif method == 'tsne':
        pass
    else:
        print("Method is not available. use 'pca' (default) or 'tsne'.")
        return
    visualize_dim_red(r_dim_red, labels, filename, figsize, title, **scatter_options)


Instruction:
Create a Python 3 function to
visualize the dimred matrix r.

Response:
def visualize_dim_red(r, labels, filename=None, figsize=(18,10), title='', legend=True, label_map=None, label_scale=False, label_color_map=None, **scatter_options):
    """
    Saves a scatter plot of a (2,n) matrix r, where each column is a cell.

    Args:
        r (array): (2,n) matrix
        labels (array): (n,) array of ints/strings or floats. Can be None.
        filename (string): string to save the output graph. If None, then this just displays the plot.
        figsize (tuple): Default: (18, 10)
        title (string): graph title
        legend (bool): Default: True
        label_map (dict): map of labels to label names. Default: None
        label_scale (bool): True if labels is should be treated as floats. Default: False
        label_color_map (array): (n,) array or list of colors for each label.
    """
    fig = plt.figure(figsize=figsize)
    plt.cla()
    if not label_scale:
        for i in set(labels):
            label = i
            if label_map is not None:
                label = label_map[i]
            if label_color_map is not None:
                c = label_color_map[i]
                plt.scatter(r[0, labels==i], r[1, labels==i], label=label, c=c, **scatter_options)
            else:
                plt.scatter(r[0, labels==i], r[1, labels==i], label=label, **scatter_options)
    else:
        if labels is None:
            plt.scatter(r[0,:], r[1,:], **scatter_options)
        else:
            plt.scatter(r[0,:], r[1,:], c=labels/labels.max(), **scatter_options)
    plt.title(title)
    if legend:
        plt.legend()
    if filename is not None:
        plt.savefig(filename, dpi=100)
        plt.close()
    return fig


Instruction:
How would you code a function in Python 3 to
run a pre - processing + clustering experiment on a list of methods.

Response:
def run_experiment(methods, data, n_classes, true_labels, n_runs=10, use_purity=True, use_nmi=False, use_ari=False, use_nne=False, consensus=False):
    """
    runs a pre-processing + clustering experiment...

    exactly one of use_purity, use_nmi, or use_ari can be true

    Args:
        methods: list of 2-tuples. The first element is either a single Preprocess object or a list of Preprocess objects, to be applied in sequence to the data. The second element is either a single Cluster object, a list of Cluster objects, or a list of lists, where each list is a sequence of Preprocess objects with the final element being a Cluster object.
        data: genes x cells array
        true_labels: 1d array of length cells
        consensus: if true, runs a consensus on cluster results for each method at the very end.
        use_purity, use_nmi, use_ari, use_nne: which error metric to use (at most one can be True)

    Returns:
        purities (list of lists)
        names (list of lists)
        other (dict): keys: timing, preprocessing, clusterings
    """
    results = []
    names = []
    clusterings = {}
    other_results = {}
    other_results['timing'] = {}
    other_results['preprocessing'] = {}
    if use_purity:
        purity_method = purity
    elif use_nmi:
        purity_method = nmi
    elif use_ari:
        purity_method = ari
    elif use_nne:
        purity_method = nne
    for i in range(n_runs):
        print('run {0}'.format(i))
        purities = []
        r = 0
        method_index = 0
        for preproc, cluster in methods:
            t0 = time.time()
            if isinstance(preproc, Preprocess):
                preprocessed, ll = preproc.run(data)
                output_names = preproc.output_names
            else:
                # if the input is a list, only use the first preproc result
                p1 = data
                output_names = ['']
                for p in preproc:
                    p1, ll = p.run(p1)
                    p1 = p1[0]
                    if output_names[0] != '':
                        output_names[0] = output_names[0] + '_' + p.output_names[0]
                    else:
                        output_names[0] = p.output_names[0]
                preprocessed = [p1]
            t1 = time.time() - t0
            for name, pre in zip(output_names, preprocessed):
                starting_index = method_index
                if isinstance(cluster, Cluster):
                    #try:
                        t0 = time.time()
                        labels = cluster.run(pre)
                        t2 = t1 + time.time() - t0
                        if use_nne:
                            purities.append(purity_method(pre, true_labels))
                        else:
                            purities.append(purity_method(labels, true_labels))
                        if i==0:
                            names.append(name + '_' + cluster.name)
                            clusterings[names[-1]] = []
                            other_results['timing'][names[-1]] = []
                        print(names[r])
                        clusterings[names[r]].append(labels)
                        print('time: ' + str(t2))
                        other_results['timing'][names[r]].append(t2)
                        print(purities[-1])
                        r += 1
                        method_index += 1
                    #except:
                    #    print('failed to do clustering')
                elif type(cluster) == list:
                    for c in cluster:
                        if isinstance(c, list):
                            t2 = t1
                            name2 = name
                            sub_data = pre.copy()
                            for subproc in c[:-1]:
                                t0 = time.time()
                                subproc_out, ll = subproc.run(sub_data)
                                sub_data = subproc_out[0]
                                name2 = name2 + '_' + subproc.output_names[0]
                                t2 += time.time() - t0
                            t0 = time.time()
                            labels = c[-1].run(sub_data)
                            t2 += time.time() - t0
                            if use_nne:
                                purities.append(purity_method(sub_data, true_labels))
                            else:
                                purities.append(purity_method(labels, true_labels))
                            if i==0:
                                names.append(name2 + '_' + c[-1].name)
                                clusterings[names[-1]] = []
                                other_results['timing'][names[-1]] = []
                            print(names[r])
                            clusterings[names[r]].append(labels)
                            other_results['timing'][names[r]].append(t2)
                            print('time: ' + str(t2))
                            print(purities[-1])
                            r += 1
                            method_index += 1
                        else:
                            try:
                                t0 = time.time()
                                labels = c.run(pre)
                                t2 = t1 + time.time() - t0
                                if i==0:
                                    names.append(name + '_' + c.name)
                                    clusterings[names[-1]] = []
                                    other_results['timing'][names[-1]] = []
                                if use_nne:
                                    purities.append(purity_method(pre, true_labels))
                                else:
                                    purities.append(purity_method(labels, true_labels))
                                print(names[r])
                                clusterings[names[r]].append(labels)
                                other_results['timing'][names[r]].append(t2)
                                print('time: ' + str(t2))
                                print(purities[-1])
                                r += 1
                                method_index += 1
                            except:
                                print('failed to do clustering')
                # find the highest purity for the pre-processing method
                # save the preprocessing result with the highest NMI
                num_clustering_results = method_index - starting_index
                clustering_results = purities[-num_clustering_results:]
                if i > 0 and len(clustering_results) > 0:
                    old_clustering_results = results[-1][starting_index:method_index]
                    if max(old_clustering_results) < max(clustering_results):
                        other_results['preprocessing'][name] = pre
                else:
                    other_results['preprocessing'][name] = pre
        print('\t'.join(names))
        print('purities: ' + '\t'.join(map(str, purities)))
        results.append(purities)
    consensus_purities = []
    if consensus:
        other_results['consensus'] = {}
        k = len(np.unique(true_labels))
        for name, clusts in clusterings.items():
            print(name)
            clusts = np.vstack(clusts)
            consensus_clust = CE.cluster_ensembles(clusts, verbose=False, N_clusters_max=k)
            other_results['consensus'][name] = consensus_clust
            if use_purity:
                consensus_purity = purity(consensus_clust.flatten(), true_labels)
                print('consensus purity: ' + str(consensus_purity))
                consensus_purities.append(consensus_purity)
            if use_nmi:
                consensus_nmi = nmi(true_labels, consensus_clust)
                print('consensus NMI: ' + str(consensus_nmi))
                consensus_purities.append(consensus_nmi)
            if use_ari:
                consensus_ari = ari(true_labels, consensus_clust)
                print('consensus ARI: ' + str(consensus_ari))
                consensus_purities.append(consensus_ari)
        print('consensus results: ' + '\t'.join(map(str, consensus_purities)))
    other_results['clusterings'] = clusterings
    return results, names, other_results


Instruction:
Can you generate the documentation for the following Python 3 function
def generate_visualizations(methods, data, true_labels, base_dir = 'visualizations',
        figsize=(18,10), **scatter_options):
    """
    Generates visualization scatters for all the methods.

    Args:
        methods: follows same format as run_experiments. List of tuples.
        data: genes x cells
        true_labels: array of integers
        base_dir: base directory to save all the plots
        figsize: tuple of ints representing size of figure
        scatter_options: options for plt.scatter
    """
    plt.figure(figsize=figsize)
    for method in methods:
        preproc= method[0]
        if isinstance(preproc, Preprocess):
            preprocessed, ll = preproc.run(data)
            output_names = preproc.output_names
        else:
            # if the input is a list, only use the first preproc result
            p1 = data
            output_names = ['']
            for p in preproc:
                p1, ll = p.run(p1)
                p1 = p1[0]
                output_names[0] = output_names[0] + p.output_names[0]
            preprocessed = [p1]
        for r, name in zip(preprocessed, output_names):
            # TODO: cluster labels
            print(name)
            # if it's 2d, just display it... else, do tsne to reduce to 2d
            if r.shape[0]==2:
                r_dim_red = r
            else:
                # sometimes the data is too big to do tsne... (for sklearn)
                if sparse.issparse(r) and r.shape[0] > 100:
                    name = 'tsvd_' + name
                    tsvd = TruncatedSVD(50)
                    r_dim_red = tsvd.fit_transform(r.T)
                    try:
                        tsne = TSNE(2)
                        r_dim_red = tsne.fit_transform(r_dim_red).T
                        name = 'tsne_' + name
                    except:
                        tsvd2 = TruncatedSVD(2)
                        r_dim_red = tsvd2.fit_transform(r_dim_red).T
                else:
                    name = 'tsne_' + name
                    tsne = TSNE(2)
                    r_dim_red = tsne.fit_transform(r.T).T
            if isinstance(method[1], list):
                for clustering_method in method[1]:
                    try:
                        cluster_labels = clustering_method.run(r)
                    except:
                        print('clustering failed')
                        continue
                    output_path = base_dir + '/{0}_{1}_labels.png'.format(name, clustering_method.name)
                    visualize_dim_red(r_dim_red, cluster_labels, output_path, **scatter_options)
            else:
                clustering_method = method[1]
                try:
                    cluster_labels = clustering_method.run(r)
                except:
                    print('clustering failed')
                    continue
                output_path = base_dir + '/{0}_{1}_labels.png'.format(name, clustering_method.name)
                visualize_dim_red(r_dim_red, cluster_labels, output_path, **scatter_options)
            output_path = base_dir + '/{0}_true_labels.png'.format(name)
            visualize_dim_red(r_dim_red, true_labels, output_path, **scatter_options)

Response: Generates visualizations for all the methods in the cluster.


Instruction:
Implement a Python 3 function for
running the Poisson EstimateState method.

Response:
def run(self, data):
        """
        Returns:
            list of W, M*W
            ll
        """
        if self.normalize_data:
            data = cell_normalize(data)
        M, W, ll = poisson_estimate_state(data, **self.params)
        outputs = []
        if self.return_w:
            outputs.append(W)
        if self.return_m:
            outputs.append(M)
        if self.return_mw:
            outputs.append(M.dot(W))
        if self.return_mds:
            X = dim_reduce(M, W, 2)
            outputs.append(X.T.dot(W))
        return outputs, ll


Instruction:
Can you generate the documentation for the following Python 3 function
def calculate_hex(hex_string):
        hex_conv = hex_string.split(':')
        converted_hex = (int(hex_conv[0], 16) + int(hex_conv[1], 16))/8192

        return converted_hex

Response: Credit for conversion to itsnotlupus / vesync_wsproxy


Instruction:
Can you generate the documentation for the following Python 3 function
def resolve_updates(orig_list, updated_list):

        if updated_list is not None and updated_list:
            if orig_list is None:
                orig_list = updated_list
            else:
                # Add new devices not in list but found in the update
                for new_device in updated_list:
                    was_found = False

                    for device in orig_list:
                        if new_device.cid == device.cid:
                            was_found = True
                            break

                    if not was_found:
                        orig_list.append(new_device)

                # Remove old devices in the list not found in the update
                for device in orig_list:
                    should_remove = True

                    for new_device in updated_list:
                        if device.cid == new_device.cid:
                            should_remove = False
                            break

                    if should_remove:
                        orig_list.remove(device)

            # Call update on each device in the list
            [device.update() for device in orig_list]

        return orig_list

Response: Merges changes from one list of devices against another


Instruction:
How would you explain what the following Python 3 function does
def main (args):
  if (len(args) != 2):
    print(main.__doc__)
    sys.exit(1);

  # create the document
  doc = libsedml.SedDocument();
  doc.setLevel(1);
  doc.setVersion(1);

  # create a first model referencing an sbml file
  model = doc.createModel();
  model.setId("model1");
  model.setSource("file.xml");
  model.setLanguage("urn:sedml:sbml");

  # create a second model modifying a variable of that other sbml file
  model = doc.createModel();
  model.setId("model2");
  model.setSource("model1");
  model.setLanguage("urn:sedml:sbml");

  # change a paramerter 'k' to 0.1
  change = model.createChangeAttribute();
  change.setTarget("/sbml:sbml/sbml:model/sbml:listOfParameters/sbml:parameter[@id='k']/@value");
  change.setNewValue("0.1");

  # remove species 's1'
  remove = model.createRemoveXML();
  remove.setTarget("/sbml:sbml/sbml:model/sbml:listOfSpecies/sbml:species[@id='S1']");

  # now for something tricky we want to update the initialConcentration of 'S2' to be
  # half what it was in the original model
  compute = model.createComputeChange();
  compute.setTarget("/sbml:sbml/sbml:model/sbml:listOfSpecies/sbml:species[@id="S2"]/@initialConcentration");
  variable = compute.createVariable();
  variable.setId("S2");
  variable.setModelReference("model1");
  variable.setTarget("/sbml:sbml/sbml:model/sbml:listOfSpecies/sbml:species[@id='S2']");
  compute.setMath(libsedml.parseFormula("S2 / 2"));

  # create simulation
  tc = doc.createUniformTimeCourse();
  tc.setId("sim1");
  tc.setInitialTime(0.0);
  tc.setOutputStartTime(0.0);
  tc.setOutputEndTime(10.0);
  tc.setNumberOfPoints(1000);
  # need to set the correct KISAO Term
  alg = tc.createAlgorithm();
  alg.setKisaoID("KISAO:0000019");

  # create a task that uses the simulation and the model above
  task = doc.createTask();
  task.setId("task1");
  task.setModelReference("model1");
  task.setSimulationReference("sim1");

  # add a DataGenerator to hold the output for time
  dg = doc.createDataGenerator();
  dg.setId("time");
  dg.setName("time");
  var = dg.createVariable();
  var.setId("v0");
  var.setName("time");
  var.setTaskReference("task1");
  var.setSymbol("urn:sedml:symbol:time");
  dg.setMath(libsedml.parseFormula("v0"));

  # and one for S1
  dg = doc.createDataGenerator();
  dg.setId("S1");
  dg.setName("S1");
  var = dg.createVariable();
  var.setId("v1");
  var.setName("S1");
  var.setTaskReference("task1");
  var.setTarget("/sbml:sbml/sbml:model/sbml:listOfSpecies/sbml:species[@id='S1']");
  dg.setMath(libsedml.parseFormula("v1"));

  # add a report
  report = doc.createReport();
  report.setId("r1");
  report.setName("report 1");
  set = report.createDataSet();
  set.setId("ds1");
  set.setLabel("time");
  set.setDataReference("time");
  set = report.createDataSet();
  set.setId("ds2");
  set.setLabel("S1");
  set.setDataReference("S1");

  # add a 2d plot
  plot = doc.createPlot2D();
  plot.setId("p1");
  plot.setName("S1 Timecourse");
  curve = plot.createCurve();
  curve.setId("c1");
  curve.setName("S1");
  curve.setLogX(False);
  curve.setLogY(False);
  curve.setXDataReference("time");
  curve.setYDataReference("S1");

  # add a 3D Plot
  plot2 = doc.createPlot3D();
  plot2.setId("p2");
  plot2.setName("dunno");
  surf = plot2.createSurface();
  surf.setId("surf1");
  surf.setName("S1");
  surf.setLogX(False);
  surf.setLogY(False);
  surf.setLogZ(False);
  surf.setXDataReference("time");
  surf.setYDataReference("S1");
  surf.setZDataReference("S1");

  # write the document
  libsedml.writeSedML(doc, args[1]);

Response: create_sedml output - filename


Instruction:
Can you implement a function in Python 3 that
gives a vcf file return a dictionary of profile strings for each sample in the vcf

Response:
def get_profiles(adapter, vcf_file):

    """Given a vcf, get a profile string for each sample in the vcf
    based on the profile variants in the database

    Args:
        adapter(MongoAdapter): Adapter to mongodb
        vcf_file(str): Path to vcf file

    Returns:
        profiles (dict(str)): The profiles (given as strings) for each sample
                              in vcf.

    """
    vcf = get_file_handle(vcf_file)
    individuals = vcf.samples
    profiles = {individual: [] for individual in individuals}

    for profile_variant in adapter.profile_variants():

        ref = profile_variant['ref']
        alt = profile_variant['alt']

        pos = profile_variant['pos']
        end = pos + 1
        chrom = profile_variant['chrom']

        region = f"{chrom}:{pos}-{end}"

        #Find variants in region

        found_variant = False
        for variant in vcf(region):

            variant_id = get_variant_id(variant)

            #If variant id i.e. chrom_pos_ref_alt matches
            if variant_id == profile_variant['_id']:
                found_variant = True
                #find genotype for each individual in vcf
                for i, individual in enumerate(individuals):

                    genotype = GENOTYPE_MAP[variant.gt_types[i]]
                    if genotype == 'hom_alt':
                        gt_str = f"{alt}{alt}"
                    elif genotype == 'het':
                        gt_str = f"{ref}{alt}"
                    else:
                        gt_str = f"{ref}{ref}"

                    #Append genotype to profile string of individual
                    profiles[individual].append(gt_str)

                #Break loop if variant is found in region
                break

        #If no call was found for variant, give all samples a hom ref genotype
        if not found_variant:
            for individual in individuals: profiles[individual].append(f"{ref}{ref}")

    return profiles


Instruction:
Write a Python 3 script for
returning a list of similar samples for each sample in the DB.

Response:
def profile_match(adapter, profiles, hard_threshold=0.95, soft_threshold=0.9):

    """
        given a dict of profiles, searches through all the samples in the DB
        for a match. If a matching sample is found an exception is raised,
        and the variants will not be loaded into the database.

        Args:
            adapter (MongoAdapter): Adapter to mongodb
            profiles (dict(str)): The profiles (given as strings) for each sample in vcf.
            hard_threshold(float): Rejects load if hamming distance above this is found
            soft_threshold(float): Stores similar samples if hamming distance above this is found

        Returns:
            matches(dict(list)): list of similar samples for each sample in vcf.
    """
    matches = {sample: [] for sample in profiles.keys()}
    for case in adapter.cases():

        for individual in case['individuals']:

            for sample in profiles.keys():

                if individual.get('profile'):

                    similarity = compare_profiles(
                        profiles[sample], individual['profile']
                    )

                    if similarity >= hard_threshold:

                        msg = (
                                f"individual {sample} has a {similarity} similarity "
                                f"with individual {individual['ind_id']} in case "
                                f"{case['case_id']}"
                        )
                        LOG.critical(msg)

                        #Raise some exception
                        raise ProfileError

                    if similarity >= soft_threshold:

                        match = f"{case['case_id']}.{individual['ind_id']}"
                        matches[sample].append(match)

    return matches


Instruction:
How would you implement a function in Python 3 that
gives two profiles, determine the ratio of similarity, i.e. the hamming distance between the strings. Args: profile1/2 (str): profile string Returns: similarity_ratio (float): the ratio of similiarity (0-1)

Response:
def compare_profiles(profile1, profile2):

    """
        Given two profiles, determine the ratio of similarity, i.e.
        the hamming distance between the strings.

        Args:
            profile1/2 (str): profile string
        Returns:
            similarity_ratio (float): the ratio of similiarity (0-1)
    """

    length = len(profile1)

    profile1 = np.array(list(profile1))
    profile2 = np.array(list(profile2))

    similarity_array = profile1 == profile2

    matches = np.sum(similarity_array)

    similarity_ratio = matches/length

    return similarity_ratio


Instruction:
Create a Python 3 function for
updating the profile string for all samples in the specified vcf

Response:
def update_profiles(adapter):

    """
    For all cases having vcf_path, update the profile string for the samples

    Args:
        adapter (MongoAdapter): Adapter to mongodb

    """

    for case in adapter.cases():

        #If the case has a vcf_path, get the profiles and update the
        #case with new profiled individuals.
        if case.get('profile_path'):

            profiles = get_profiles(adapter, case['profile_path'])
            profiled_individuals = deepcopy(case['individuals'])

            for individual in profiled_individuals:
                ind_id = individual['ind_id']
                try:
                    profile = profiles[ind_id]
                    individual['profile'] = profile

                except KeyError:
                    LOG.warning(f"sample IDs in vcf does not match for case {case['case_id']}")

            updated_case = deepcopy(case)

            updated_case['individuals'] = profiled_individuals

            adapter.add_case(updated_case, update=True)


Instruction:
Write a Python 3 function that can
return a dictionary with the number of hamming distances for all the sample profiles in the database.

Response:
def profile_stats(adapter, threshold = 0.9):

    """
        Compares the pairwise hamming distances for all the sample profiles in
        the database. Returns a table of the number of distances within given
        ranges.

        Args:
            adapter (MongoAdapter): Adapter to mongodb
            threshold (float): If any distance is found above this threshold
                a warning will be given, stating the two matching samples.

        Returns:
            distance_dict (dict): dictionary with ranges as keys, and the number
                of distances that are within these ranges as values.

    """
    profiles = []
    samples = []

    #Instatiate the distance dictionary with a count 0 for all the ranges
    distance_dict = {key: 0 for key in HAMMING_RANGES.keys()}

    for case in adapter.cases():

        for individual in case['individuals']:

            if individual.get('profile'):
                #Make sample name <case_id>.<sample_id>
                sample_id = f"{case['case_id']}.{individual['ind_id']}"
                ind_profile = individual['profile']

                #Numpy array to hold all the distances for this samples profile
                distance_array = np.array([], dtype=np.float)

                for sample, profile in zip(samples, profiles):

                    #Get distance and append to distance array
                    distance = compare_profiles(ind_profile, profile)
                    distance_array = np.append(distance_array, distance)

                    #Issue warning if above threshold
                    if distance >= threshold:
                        LOG.warning(f"{sample_id} is {distance} similar to {sample}")

                #Check number of distances in each range and add to distance_dict
                for key,range in HAMMING_RANGES.items():

                    #Calculate the number of hamming distances found within the
                    #range for current individual
                    distance_dict[key] += np.sum(
                        (distance_array >= range[0]) & (distance_array < range[1])
                    )

                #Append profile and sample_id for this sample for the next
                #iteration
                profiles.append(ind_profile)
                samples.append(sample_id)

    return distance_dict


Instruction:
Can you implement a function in Python 3 that
calculates the purity score for the given labels.

Response:
def purity(labels, true_labels):
    """
    Calculates the purity score for the given labels.

    Args:
        labels (array): 1D array of integers
        true_labels (array): 1D array of integers - true labels

    Returns:
        purity score - a float bewteen 0 and 1. Closer to 1 is better.
    """
    purity = 0.0
    for i in set(labels):
        indices = (labels==i)
        true_clusters = true_labels[indices]
        if len(true_clusters)==0:
            continue
        counts = Counter(true_clusters)
        lab, count = counts.most_common()[0]
        purity += count
    return float(purity)/len(labels)


Instruction:
Can you tell what is the following Python 3 function doing
def mdl(ll, k, data):

    """
    N - no. of genes
    n - no. of cells
    k - no. of cell types
    R - sum(Dataset) i.e. total no. of reads

    function TotCost = TotBits(N,m,p,R,C)
        # C is the cost from the cost function
        TotCost = C + (N*m + m*p)*(log(R/(N*p)));
    """
    N, m = data.shape
    cost = ll + (N*m + m*k)*(np.log(data.sum()/(N*k)))
    return cost

Response: Returns the minimum description length score of the model given its log - likelihood and k the number of cell types.


Instruction:
Make a summary of the following Python 3 code
def find_nb_genes(data):
    data_means = data.mean(1)
    data_vars = data.var(1)
    nb_indices = data_means < 0.9*data_vars
    return nb_indices

Response: Finds the indices of all genes in the dataset that have a mean < 0. 9 variance. Returns an array of booleans.


Instruction:
Make a summary of the following Python 3 code
def log_ncr(a, b):
    val = gammaln(a+1) - gammaln(a-b+1) - gammaln(b+1)
    return val

Response: Returns the log of the n - cr function given that a < b.


Instruction:
Explain what the following Python 3 code does
def nb_ll(data, P, R):
    # TODO: include factorial...
    #data = data + eps
    genes, cells = data.shape
    clusters = P.shape[1]
    lls = np.zeros((cells, clusters))
    for c in range(clusters):
        P_c = P[:,c].reshape((genes, 1))
        R_c = R[:,c].reshape((genes, 1))
        # don't need constant factors...
        ll = gammaln(R_c + data) - gammaln(R_c) #- gammaln(data + 1)
        ll += data*np.log(P_c) + xlog1py(R_c, -P_c)
        #new_ll = np.sum(nbinom.logpmf(data, R_c, P_c), 0)
        lls[:,c] = ll.sum(0)
    return lls

Response: Returns the negative binomial log - likelihood of the data.


Instruction:
How would you explain what the following Python 3 function does
def zinb_ll(data, P, R, Z):
    lls = nb_ll(data, P, R)
    clusters = P.shape[1]
    for c in range(clusters):
        pass
    return lls

Response: Returns the zero - inflated negative binomial log - likelihood of the data.


Instruction:
Here you have a function in Python 3, explain what it does
def nb_ll_row(params, data_row):
    p = params[0]
    r = params[1]
    n = len(data_row)
    ll = np.sum(gammaln(data_row + r)) - np.sum(gammaln(data_row + 1))
    ll -= n*gammaln(r)
    ll += np.sum(data_row)*np.log(p)
    ll += n*r*np.log(1-p)
    return -ll

Response: Returns the negative LL of a single row.


Instruction:
Given the following Python 3 function, write the documentation
def nb_fit(data, P_init=None, R_init=None, epsilon=1e-8, max_iters=100):
    means = data.mean(1)
    variances = data.var(1)
    if (means > variances).any():
        raise ValueError("For NB fit, means must be less than variances")
    genes, cells = data.shape
    # method of moments
    P = 1.0 - means/variances
    R = means*(1-P)/P
    for i in range(genes):
        result = minimize(nb_ll_row, [P[i], R[i]], args=(data[i,:],),
                bounds = [(0, 1), (eps, None)])
        params = result.x
        P[i] = params[0]
        R[i] = params[1]
        #R[i] = fsolve(nb_r_deriv, R[i], args = (data[i,:],))
        #P[i] = data[i,:].mean()/(data[i,:].mean() + R[i])
    return P,R

Response: Fits the NB distribution to data using method of moments.


Instruction:
Can you generate the documentation for the following Python 3 function
def nb_cluster(data, k, P_init=None, R_init=None, assignments=None, means=None, max_iters=10):
    genes, cells = data.shape
    if P_init is None:
        P_init = np.random.random((genes, k))
    if R_init is None:
        R_init = np.random.randint(1, data.max(), (genes, k))
        R_init = R_init.astype(float)
    if assignments is None:
        _, assignments = kmeans_pp(data, k, means)
    means = np.zeros((genes, k))
        #assignments = np.array([np.random.randint(0,k) for i in range(cells)])
    old_assignments = np.copy(assignments)
    # If mean > variance, then fall back to Poisson, since NB
    # distribution can't handle that case.
    for i in range(max_iters):
        # estimate params from assigned cells
        nb_gene_indices = fit_cluster(data, assignments, k, P_init, R_init, means)
        # re-calculate assignments
        lls = nb_ll(data[nb_gene_indices, :], P_init[nb_gene_indices,:], R_init[nb_gene_indices,:])
        lls += pois_ll.poisson_ll(data[~nb_gene_indices,:], means[~nb_gene_indices,:])
        # set NB params to failure values
        P_init[~nb_gene_indices,:] = 0
        R_init[~nb_gene_indices,:] = np.inf
        for c in range(cells):
            assignments[c] = np.argmax(lls[c,:])
        if np.equal(assignments,old_assignments).all():
            break
        old_assignments = np.copy(assignments)
    return assignments, P_init, R_init

Response: This function performs negative binomial clustering on the given data.


Instruction:
Here you have a function in Python 3, explain what it does
def fit_cluster(data, assignments, k, P_init, R_init, means):
    for c in range(k):
        if data[:,assignments==c].shape[1] == 0:
            _, assignments = kmeans_pp(data, k)
    genes, cells = data.shape
    nb_gene_indices = np.array([True for i in range(genes)])
    for c in range(k):
        c_data = data[:,assignments==c]
        nb_gene_indices = nb_gene_indices & find_nb_genes(c_data)
    for c in range(k):
        c_data = data[:,assignments==c]
        nb_genes = c_data[nb_gene_indices,:]
        poisson_genes = c_data[~nb_gene_indices, :]
        P_init[nb_gene_indices, c], R_init[nb_gene_indices, c] = nb_fit(nb_genes)
        means[~nb_gene_indices, c] = poisson_genes.mean(1)
    return nb_gene_indices

Response: Fits NB / poisson params to a cluster.


Instruction:
Create a Python 3 function for
calculating the zero - inflated Poisson log - likelihood for each cell in the cluster.

Response:
def zip_ll(data, means, M):
    """
    Calculates the zero-inflated Poisson log-likelihood.

    Args:
        data (array): genes x cells
        means (array): genes x k
        M (array): genes x k - this is the zero-inflation parameter.

    Returns:
        cells x k array of log-likelihood for each cell/cluster pair.
    """
    genes, cells = data.shape
    clusters = means.shape[1]
    ll = np.zeros((cells, clusters))
    d0 = (data==0)
    d1 = (data>0)
    for i in range(clusters):
        means_i = np.tile(means[:,i], (cells, 1))
        means_i = means_i.transpose()
        L_i = np.tile(M[:,i], (cells, 1))
        L_i = L_i.transpose()
        ll_0 = np.log(L_i + (1 - L_i)*np.exp(-means_i))
        ll_0 = np.where((L_i==0) & (means_i==0), -means_i, ll_0)
        # not including constant factors
        ll_1 = np.log(1 - L_i) + xlogy(data, means_i) -  means_i
        ll_0 = np.where(d0, ll_0, 0.0)
        ll_1 = np.where(d1, ll_1, 0.0)
        ll[:,i] = np.sum(ll_0 + ll_1, 0)
    return ll


Instruction:
Can you write a function in Python 3 where it
returns the negative log - likelihood of a row given ZIP data.

Response:
def zip_ll_row(params, data_row):
    """
    Returns the negative log-likelihood of a row given ZIP data.

    Args:
        params (list): [lambda zero-inf]
        data_row (array): 1d array

    Returns:
        negative log-likelihood
    """
    l = params[0]
    pi = params[1]
    d0 = (data_row==0)
    likelihood = d0*pi + (1-pi)*poisson.pmf(data_row, l)
    return -np.log(likelihood+eps).sum()


Instruction:
Can you write a function in Python 3 where it
migrates an old loqusdb instance to 1. 0 WorkItem

Response:
def migrate_database(adapter):
    """Migrate an old loqusdb instance to 1.0

    Args:
        adapter

    Returns:
        nr_updated(int): Number of variants that where updated
    """

    all_variants = adapter.get_variants()
    nr_variants = all_variants.count()
    nr_updated = 0
    with progressbar(all_variants, label="Updating variants", length=nr_variants) as bar:
        for variant in bar:
            # Do not update if the variants have the correct format
            if 'chrom' in variant:
                continue
            nr_updated += 1
            splitted_id = variant['_id'].split('_')

            chrom = splitted_id[0]
            start = int(splitted_id[1])
            ref = splitted_id[2]
            alt = splitted_id[3]

            # Calculate end
            end = start + (max(len(ref), len(alt)) - 1)

            adapter.db.variant.find_one_and_update(
                {'_id': variant['_id']},
                {
                    '$set': {
                        'chrom': chrom,
                        'start': start,
                        'end': end
                    }
                }
            )

    return nr_updated


Instruction:
Write a Python 3 function for
calculating the bounding box of each node in each cluster.

Response:
def calculate_bounding_box(data):
    """
    Returns a 2 x m array indicating the min and max along each
    dimension.
    """
    mins = data.min(0)
    maxes = data.max(0)
    return mins, maxes


Instruction:
Write a Python 3 script to
calculate the gap score and sk for a given set of clustered income classes.

Response:
def calculate_gap(data, clustering, km, B=50, **kwargs):
    """
    See: https://datasciencelab.wordpress.com/2013/12/27/finding-the-k-in-k-means-clustering/

    https://web.stanford.edu/~hastie/Papers/gap.pdf

    Returns two results: the gap score, and s_k.
    """
    k = len(set(clustering))
    Wk = km.inertia_
    mins, maxes = calculate_bounding_box(data)
    Wk_est = []
    for i in range(B):
        data_sample = (maxes-mins)*np.random.random(data.shape) + mins
        km_b = KMeans(k)
        km_b.fit_predict(data_sample)
        Wk_est.append(km_b.inertia_)
    Wk_est = np.log(np.array(Wk_est))
    Wk_mean = np.mean(Wk_est)
    Wk_std = np.std(Wk_est)
    gap = Wk_mean - np.log(Wk)
    sk = np.sqrt(1 + 1.0/B)*Wk_std
    return gap, sk


Instruction:
Can you create a Python 3 function that
runs gap score for all k from k_min to k_max.

Response:
def run_gap_k_selection(data, k_min=1, k_max=50, B=5,
        skip=5, **kwargs):
    """
    Runs gap score for all k from k_min to k_max.
    """
    if k_min == k_max:
        return k_min
    gap_vals = []
    sk_vals = []
    k_range = list(range(k_min, k_max, skip))
    min_k = 0
    min_i = 0
    for i, k in enumerate(k_range):
        km = KMeans(k)
        clusters = km.fit_predict(data)
        gap, sk = calculate_gap(data, clusters, km, B=B)
        if len(gap_vals) > 1:
            if gap_vals[-1] >= gap - (skip+1)*sk:
                min_i = i
                min_k = k_range[i-1]
                break
                #return k_range[-1], gap_vals, sk_vals
        gap_vals.append(gap)
        sk_vals.append(sk)
    if min_k == 0:
        min_k = k_max
    if skip == 1:
        return min_k, gap_vals, sk_vals
    gap_vals = []
    sk_vals = []
    for k in range(min_k - skip, min_k + skip):
        km = KMeans(k)
        clusters = km.fit_predict(data)
        gap, sk = calculate_gap(data, clusters, km, B=B)
        if len(gap_vals) > 1:
            if gap_vals[-1] >= gap - sk:
                min_k = k-1
                return min_k, gap_vals, sk_vals
        gap_vals.append(gap)
        sk_vals.append(sk)
    return k, gap_vals, sk_vals


Instruction:
How would you explain what the following Python 3 function does
async def _request(
            self,
            method: str,
            url: str,
            *,
            headers: dict = None,
            params: dict = None,
            json: dict = None) -> dict:
        """Make a request against the RainMachine device."""
        if not headers:
            headers = {}

        try:
            async with self._websession.request(method, url, headers=headers,
                                                params=params,
                                                json=json) as resp:
                resp.raise_for_status()
                data = await resp.json(content_type=None)
                return data
        except ClientError as err:
            raise RequestError(
                'Error requesting data from {}: {}'.format(url, err))

Response: Make a request against the RainMachine device.


Instruction:
Make a summary of the following Python 3 code
def get_devices(self) -> list:

        if not self.enabled:
            return None

        self.in_process = True

        response, _ = helpers.call_api(
            '/cloud/v1/deviceManaged/devices',
            'post',
            headers=helpers.req_headers(self),
            json=helpers.req_body(self, 'devicelist')
        )

        if response and helpers.check_response(response, 'get_devices'):
            if 'result' in response and 'list' in response['result']:
                device_list = response['result']['list']
                outlets, switches, fans = self.process_devices(device_list)
            else:
                logger.error('Device list in response not found')
        else:
            logger.error('Error retrieving device list')

        self.in_process = False

        return (outlets, switches, fans)

Response: Return list of VeSync devices


Instruction:
Implement a Python 3 function for
logging in with username and password.

Response:
def login(self) -> bool:
        """Return True if log in request succeeds"""
        user_check = isinstance(self.username, str) and len(self.username) > 0
        pass_check = isinstance(self.password, str) and len(self.password) > 0

        if user_check and pass_check:
            response, _ = helpers.call_api(
                '/cloud/v1/user/login',
                'post',
                json=helpers.req_body(self, 'login')
            )

            if response and helpers.check_response(response, 'login'):
                self.token = response['result']['token']
                self.account_id = response['result']['accountID']
                self.enabled = True

                return True
            else:
                logger.error('Error logging in with username and password')
                return False

        else:
            if user_check is False:
                logger.error('Username invalid')
            if pass_check is False:
                logger.error('Password invalid')

        return False


Instruction:
Implement a function in Python 3 to
fetch updated information about devices

Response:
def update(self):
        """Fetch updated information about devices"""

        if self.device_time_check():

            if not self.in_process:
                outlets, switches, fans = self.get_devices()

                self.outlets = helpers.resolve_updates(self.outlets, outlets)
                self.switches = helpers.resolve_updates(
                    self.switches, switches)
                self.fans = helpers.resolve_updates(self.fans, fans)

                self.last_update_ts = time.time()


Instruction:
Create a Python 3 function to
fetch updated energy information about devices

Response:
def update_energy(self, bypass_check=False):
        """Fetch updated energy information about devices"""
        for outlet in self.outlets:
            outlet.update_energy(bypass_check)


Instruction:
Implement a Python 3 function for
giving a data matrix, this returns the per-gene fit error for the Poisson, Normal, and Log-Normal distributions. Args: Dat (array): numpy array with shape (genes, cells) Returns: d (dict): 'poiss', 'norm', 'lognorm' give the fit error for each distribution.

Response:
def DistFitDataset(Dat):
    """
    Given a data matrix, this returns the per-gene fit error for the
    Poisson, Normal, and Log-Normal distributions.

    Args:
        Dat (array): numpy array with shape (genes, cells)

    Returns:
        d (dict): 'poiss', 'norm', 'lognorm' give the fit error for each distribution.
    """
    #Assumes data to be in the form of a numpy matrix
    (r,c) = Dat.shape
    Poiss = np.zeros(r)
    Norm = np.zeros(r)
    LogNorm = np.zeros(r)
    for i in range(r):
        temp = GetDistFitError(Dat[i])
        Poiss[i] = temp['poiss']
        Norm[i] = temp['norm']
        LogNorm[i] = temp['lognorm']
    d = {}
    d['poiss'] = Poiss
    d['norm'] = Norm
    d['lognorm'] = LogNorm
    return d


Instruction:
Can you write a function in Python 3 where it
deletes a case and all of its variants from the database.

Response:
def delete(adapter, case_obj, update=False, existing_case=False):
    """Delete a case and all of it's variants from the database.

    Args:
        adapter: Connection to database
        case_obj(models.Case)
        update(bool): If we are in the middle of an update
        existing_case(models.Case): If something failed during an update we need to revert
                                    to the original case

    """
    # This will overwrite the updated case with the previous one
    if update:
        adapter.add_case(existing_case)
    else:
        adapter.delete_case(case_obj)

    for file_type in ['vcf_path','vcf_sv_path']:
        if not case_obj.get(file_type):
            continue
        variant_file = case_obj[file_type]
        # Get a cyvcf2.VCF object
        vcf_obj = get_vcf(variant_file)

        delete_variants(
            adapter=adapter,
            vcf_obj=vcf_obj,
            case_obj=case_obj,
        )


Instruction:
Implement a Python 3 function for
deleting variants for a given VCF file.

Response:
def delete_variants(adapter, vcf_obj, case_obj, case_id=None):
    """Delete variants for a case in the database

    Args:
        adapter(loqusdb.plugins.Adapter)
        vcf_obj(iterable(dict))
        ind_positions(dict)
        case_id(str)

    Returns:
        nr_deleted (int): Number of deleted variants
    """
    case_id = case_id or case_obj['case_id']
    nr_deleted = 0
    start_deleting = datetime.now()
    chrom_time = datetime.now()
    current_chrom = None
    new_chrom = None

    for variant in vcf_obj:
        formated_variant = build_variant(
            variant=variant,
            case_obj=case_obj,
            case_id=case_id,
        )

        if not formated_variant:
            continue

        new_chrom = formated_variant.get('chrom')
        adapter.delete_variant(formated_variant)
        nr_deleted += 1

        if not current_chrom:
            LOG.info("Start deleting chromosome {}".format(new_chrom))
            current_chrom = new_chrom
            chrom_time = datetime.now()
            continue

        if new_chrom != current_chrom:
            LOG.info("Chromosome {0} done".format(current_chrom))
            LOG.info("Time to delete chromosome {0}: {1}".format(
                current_chrom, datetime.now()-chrom_time))
            LOG.info("Start deleting chromosome {0}".format(new_chrom))
            current_chrom = new_chrom

    return nr_deleted


Instruction:
How would you implement a function in Python 3 that
annotates the variants in a VCF

Response:
def annotate(ctx, variant_file, sv):
    """Annotate the variants in a VCF

    """
    adapter = ctx.obj['adapter']

    variant_path = os.path.abspath(variant_file)

    expected_type = 'snv'
    if sv:
        expected_type = 'sv'

    if 'sv':
        nr_cases = adapter.nr_cases(sv_cases=True)
    else:
        nr_cases = adapter.nr_cases(snv_cases=True)
    LOG.info("Found {0} {1} cases in database".format(nr_cases, expected_type))

    vcf_obj = get_file_handle(variant_path)
    add_headers(vcf_obj, nr_cases=nr_cases, sv=sv)
    # Print the headers
    for header_line in vcf_obj.raw_header.split('\n'):
        if len(header_line) == 0:
            continue
        click.echo(header_line)

    start_inserting = datetime.now()

    if sv:
        annotated_variants = annotate_svs(adapter, vcf_obj)
    else:
        annotated_variants = annotate_snvs(adapter, vcf_obj)
    # try:
    for variant in annotated_variants:
        click.echo(str(variant).rstrip())


Instruction:
Can you generate a brief explanation for the following Python 3 code
async def find(self, *tracking_numbers: str) -> list:
        data = {'data': [{'num': num} for num in tracking_numbers]}
        tracking_resp = await self._request('post', API_URL_TRACK, json=data)

        print(tracking_resp)

        if not tracking_resp.get('dat'):
            raise InvalidTrackingNumberError('Invalid data')

        packages = []
        for info in tracking_resp['dat']:
            package_info = info.get('track', {})

            if not package_info:
                continue

            kwargs = {
                'destination_country': package_info.get('c'),
                'info_text': package_info.get('z0', {}).get('z'),
                'location': package_info.get('z0', {}).get('c'),
                'origin_country': package_info.get('b'),
                'package_type': package_info.get('d', 0),
                'status': package_info.get('e', 0),
                'tracking_info_language': package_info.get('ln1', 'Unknown')
            }
            packages.append(Package(info['no'], **kwargs))
        return packages

Response: Get tracking info for one or more tracking numbers.


Instruction:
Can you tell what is the following Python 3 function doing
def binarize(qualitative):
    thresholds = qualitative.min(1) + (qualitative.max(1) - qualitative.min(1))/2.0
    binarized = qualitative > thresholds.reshape((len(thresholds), 1)).repeat(8,1)
    return binarized.astype(int)

Response: binarizes an expression dataset.


Instruction:
Implement a Python 3 function for
doing qualNorm but returns a filtered gene set based on a p - value threshold.

Response:
def qualNorm_filter_genes(data, qualitative, pval_threshold=0.05, smoothing=1e-5, eps=1e-5):
    """
    Does qualNorm but returns a filtered gene set, based on a p-value threshold.
    """
    genes, cells = data.shape
    clusters = qualitative.shape[1]
    output = np.zeros((genes, clusters))
    missing_indices = []
    genes_included = []
    qual_indices = []
    thresholds = qualitative.min(1) + (qualitative.max(1) - qualitative.min(1))/2.0
    pvals = np.zeros(genes)
    for i in range(genes):
        if qualitative[i,:].max() == -1 and qualitative[i,:].min() == -1:
            missing_indices.append(i)
            continue
        qual_indices.append(i)
        threshold = thresholds[i]
        data_i = data[i,:]
        if sparse.issparse(data):
            data_i = data_i.toarray().flatten()
        assignments, means = poisson_cluster(data_i.reshape((1, cells)), 2)
        means = means.flatten()
        high_i = 1
        low_i = 0
        if means[0]>means[1]:
            high_i = 0
            low_i = 1
        # do a p-value test
        p_val = poisson_test(data_i[assignments==low_i], data_i[assignments==high_i], smoothing=smoothing)
        pvals[i] = p_val
        if p_val <= pval_threshold:
            genes_included.append(i)
        else:
            continue
        high_mean = np.median(data_i[assignments==high_i])
        low_mean = np.median(data_i[assignments==low_i]) + eps
        for k in range(clusters):
            if qualitative[i,k]>threshold:
                output[i,k] = high_mean
            else:
                output[i,k] = low_mean
    output = output[genes_included,:]
    pvals = pvals[genes_included]
    return output, pvals, genes_included


Instruction:
Explain what the following Python 3 code does
def qualNorm(data, qualitative):
    genes, cells = data.shape
    clusters = qualitative.shape[1]
    output = np.zeros((genes, clusters))
    missing_indices = []
    qual_indices = []
    thresholds = qualitative.min(1) + (qualitative.max(1) - qualitative.min(1))/2.0
    for i in range(genes):
        if qualitative[i,:].max() == -1 and qualitative[i,:].min() == -1:
            missing_indices.append(i)
            continue
        qual_indices.append(i)
        threshold = thresholds[i]
        data_i = data[i,:]
        if sparse.issparse(data):
            data_i = data_i.toarray().flatten()
        assignments, means = poisson_cluster(data_i.reshape((1, cells)), 2)
        means = means.flatten()
        high_i = 1
        low_i = 0
        if means[0]>means[1]:
            high_i = 0
            low_i = 1
        high_mean = np.median(data_i[assignments==high_i])
        low_mean = np.median(data_i[assignments==low_i])
        for k in range(clusters):
            if qualitative[i,k]>threshold:
                output[i,k] = high_mean
            else:
                output[i,k] = low_mean
    if missing_indices:
        assignments, means = poisson_cluster(data[qual_indices, :], clusters, output[qual_indices, :], max_iters=1)
        for ind in missing_indices:
            for k in range(clusters):
                if len(assignments==k)==0:
                    output[ind, k] = data[ind,:].mean()
                else:
                    output[ind, k] = data[ind, assignments==k].mean()
    return output

Response: Generates starting positions for state estimation or qualitative data.


Instruction:
Can you generate the documentation for the following Python 3 function
def qualNormGaussian(data, qualitative):
    genes, cells = data.shape
    clusters = qualitative.shape[1]
    output = np.zeros((genes, clusters))
    missing_indices = []
    qual_indices = []
    for i in range(genes):
        if qualitative[i,:].max() == -1 and qualitative[i,:].min() == -1:
            missing_indices.append(i)
            continue
        qual_indices.append(i)
        threshold = (qualitative[i,:].max() - qualitative[i,:].min())/2.0
        kmeans = KMeans(n_clusters = 2).fit(data[i,:].reshape((1, cells)))
        assignments = kmeans.labels_
        means = kmeans.cluster_centers_
        high_mean = means.max()
        low_mean = means.min()
        for k in range(clusters):
            if qualitative[i,k]>threshold:
                output[i,k] = high_mean
            else:
                output[i,k] = low_mean
    if missing_indices:
        #generating centers for missing indices
        M_init = output[qual_indices, :]
        kmeans = KMeans(n_clusters = 2, init = M_init, max_iter = 1).fit(data[qual_indices, :])
        assignments = kmeans.labels_
        #assignments, means = poisson_cluster(data[qual_indices, :], clusters, output[qual_indices, :], max_iters=1)
        for ind in missing_indices:
            for k in range(clusters):
                output[ind, k] = np.mean(data[ind, assignments==k])
    # TODO: assign to closest
    return output

Response: Generates starting positions for state estimation or qualitative data for a given gene.


Instruction:
Write a Python 3 function that can
get the directory of the current script s contents.

Response:
def script_dir(pyobject, follow_symlinks=True):
    """Get current script's directory

    Args:
        pyobject (Any): Any Python object in the script
        follow_symlinks (Optional[bool]): Follow symlinks or not. Defaults to True.

    Returns:
        str: Current script's directory
    """
    if getattr(sys, 'frozen', False):  # py2exe, PyInstaller, cx_Freeze
        path = abspath(sys.executable)
    else:
        path = inspect.getabsfile(pyobject)
    if follow_symlinks:
        path = realpath(path)
    return dirname(path)


Instruction:
How would you explain what the following Python 3 function does
def script_dir_plus_file(filename, pyobject, follow_symlinks=True):
    return join(script_dir(pyobject, follow_symlinks), filename)

Response: Get current script s directory and then append a filename


Instruction:
How would you explain what the following Python 3 function does
def identity(ctx, variant_id):
    if not variant_id:
        LOG.warning("Please provide a variant id")
        ctx.abort()

    adapter = ctx.obj['adapter']
    version = ctx.obj['version']

    LOG.info("Search variants {0}".format(adapter))

    result = adapter.get_clusters(variant_id)
    if result.count() == 0:
        LOG.info("No hits for variant %s", variant_id)
        return

    for res in result:
        click.echo(res)

Response: Check how well SVs are working in the database


Instruction:
Can you generate the documentation for the following Python 3 function
def registration_authority_entity_id(self):

        if ATTR_ENTITY_REGISTRATION_AUTHORITY in self.raw:
            try:
                return self.raw[
                    ATTR_ENTITY_REGISTRATION_AUTHORITY][
                    ATTR_ENTITY_REGISTRATION_AUTHORITY_ENTITY_ID][
                    ATTR_DOLLAR_SIGN]
            except KeyError:
                pass

Response: Return the register entity id for the given resource.


Instruction:
Write a Python 3 function that can
return the ELF - code for the specified resource.

Response:
def legal_form(self):
        """In some cases, the legal form is stored in the JSON-data.
        In other cases, an ELF-code, consisting of mix of exactly
        four letters and numbers are stored. This ELF-code
        can be looked up in a registry where a code maps to
        a organizational type. ELF-codes are not unique,
        it can reoccur under different names in different
        countries"""

        if ATTR_ENTITY_LEGAL_FORM in self.raw:
            try:
                return LEGAL_FORMS[self.legal_jurisdiction][
                    self.raw[ATTR_ENTITY_LEGAL_FORM][
                        ATTR_ENTITY_LEGAL_FORM_CODE][ATTR_DOLLAR_SIGN]
                ]
            except KeyError:
                legal_form = self.raw[
                    ATTR_ENTITY_LEGAL_FORM][
                    ATTR_ENTITY_LEGAL_FORM_CODE][ATTR_DOLLAR_SIGN]

                if len(legal_form) == 4:
                    # If this is returned, the ELF should
                    # be added to the constants.
                    return 'ELF code: ' + legal_form
                else:
                    return legal_form


Instruction:
Make a summary of the following Python 3 code
def valid_child_records(self):
        child_lei = list()

        """Loop through data to find a valid record. Return list of LEI."""
        for d in self.raw['data']:

            # We're not very greedy here, but it seems some records have
            # lapsed even through the issuer is active
            if d['attributes']['relationship']['status'] in ['ACTIVE']:
                child_lei.append(
                    d['attributes']['relationship']['startNode']['id'])

        return child_lei

Response: Loop through data to find a valid record. Return list of LEI.


Instruction:
Here you have a function in Python 3, explain what it does
def annotate_variant(variant, var_obj=None):
    if var_obj:

        variant.INFO['Obs'] = var_obj['observations']
        if var_obj.get('homozygote'):
            variant.INFO['Hom'] = var_obj['homozygote']
        if var_obj.get('hemizygote'):
            variant.INFO['Hem'] = var_obj['hemizygote']

    return variant

Response: Annotate a cyvcf variant with observations


Instruction:
Write a Python 3 function for
annotating an SNV variant in a new CNV record

Response:
def annotate_snv(adpter, variant):
    """Annotate an SNV/INDEL variant

    Args:
        adapter(loqusdb.plugin.adapter)
        variant(cyvcf2.Variant)
    """
    variant_id = get_variant_id(variant)
    variant_obj = adapter.get_variant(variant={'_id':variant_id})

    annotated_variant = annotated_variant(variant, variant_obj)
    return annotated_variant


Instruction:
Write a Python 3 function that can
annotate all SV variants in a VCF

Response:
def annotate_svs(adapter, vcf_obj):
    """Annotate all SV variants in a VCF

    Args:
        adapter(loqusdb.plugin.adapter)
        vcf_obj(cyvcf2.VCF)

    Yields:
        variant(cyvcf2.Variant)
    """
    for nr_variants, variant in enumerate(vcf_obj, 1):
        variant_info = get_coords(variant)
        match = adapter.get_structural_variant(variant_info)
        if match:
            annotate_variant(variant, match)
        yield variant


Instruction:
Explain what the following Python 3 code does
def annotate_snvs(adapter, vcf_obj):
    variants = {}

    for nr_variants, variant in enumerate(vcf_obj, 1):
        # Add the variant to current batch
        variants[get_variant_id(variant)] = variant
        # If batch len == 1000 we annotate the batch
        if (nr_variants % 1000) == 0:

            for var_obj in adapter.search_variants(list(variants.keys())):
                var_id = var_obj['_id']
                if var_id in variants:
                    annotate_variant(variants[var_id], var_obj)

            for variant_id in variants:
                yield variants[variant_id]

            variants = {}

    for var_obj in adapter.search_variants(list(variants.keys())):
        var_id = var_obj['_id']
        if var_id in variants:
            annotate_variant(variants[var_id], var_obj)

    for variant_id in variants:
        yield variants[variant_id]

Response: Annotate all variants in a VCF in a VCF


Instruction:
Can you write a function in Python 3 where it
creates a Geometry from a GeoJSON object.

Response:
def from_geojson(geojson, srid=4326):
        """
        Create a Geometry from a GeoJSON. The SRID can be overridden from the
        expected 4326.
        """
        type_ = geojson["type"].lower()
        if type_ == "geometrycollection":
            geometries = []
            for geometry in geojson["geometries"]:
                geometries.append(Geometry.from_geojson(geometry, srid=None))
            return GeometryCollection(geometries, srid)
        elif type_ == "point":
            return Point(geojson["coordinates"], srid=srid)
        elif type_ == "linestring":
            return LineString(geojson["coordinates"], srid=srid)
        elif type_ == "polygon":
            return Polygon(geojson["coordinates"], srid=srid)
        elif type_ == "multipoint":
            geometries = _MultiGeometry._multi_from_geojson(geojson, Point)
            return MultiPoint(geometries, srid=srid)
        elif type_ == "multilinestring":
            geometries = _MultiGeometry._multi_from_geojson(geojson, LineString)
            return MultiLineString(geometries, srid=srid)
        elif type_ == "multipolygon":
            geometries = _MultiGeometry._multi_from_geojson(geojson, Polygon)
            return MultiPolygon(geometries, srid=srid)


Instruction:
How would you code a function in Python 3 to
create a Geometry from a Shapely geometry and the specified SRID.

Response:
def from_shapely(sgeom, srid=None):
        """
        Create a Geometry from a Shapely geometry and the specified SRID.

        The Shapely geometry will not be modified.
        """
        if SHAPELY:
            WKBWriter.defaults["include_srid"] = True
            if srid:
                lgeos.GEOSSetSRID(sgeom._geom, srid)
            return Geometry(sgeom.wkb_hex)
        else:
            raise DependencyError("Shapely")


Instruction:
Write a Python 3 script to
return the geometry as an E ) WKB.

Response:
def wkb(self):
        """
        Get the geometry as an (E)WKB.
        """
        return self._to_wkb(use_srid=True, dimz=self.dimz, dimm=self.dimm)


Instruction:
How would you explain what the following Python 3 function does
def postgis_type(self):
        dimz = "Z" if self.dimz else ""
        dimm = "M" if self.dimm else ""
        if self.srid:
            return "geometry({}{}{},{})".format(self.type, dimz, dimm, self.srid)
        else:
            return "geometry({}{}{})".format(self.type, dimz, dimm)

Response:
Return the type of the geometry in PostGIS format including additional additional
        dimensions and SRID.


Instruction:
Explain what the following Python 3 code does
def poisson_ll(data, means):
    if sparse.issparse(data):
        return sparse_poisson_ll(data, means)
    genes, cells = data.shape
    clusters = means.shape[1]
    ll = np.zeros((cells, clusters))
    for i in range(clusters):
        means_i = np.tile(means[:,i], (cells, 1))
        means_i = means_i.transpose() + eps
        #ll[:,i] = np.sum(xlogy(data, means_i) - gammaln(data+1) - means_i, 0)
        ll[:,i] = np.sum(xlogy(data, means_i) - means_i, 0)
    return ll

Response: Calculates the Poisson log - likelihood for each cell in the cluster pair.


Instruction:
Make a summary of the following Python 3 code
def poisson_ll_2(p1, p2):
    p1_1 = p1 + eps
    p2_1 = p2 + eps
    return np.sum(-p2_1 + p1_1*np.log(p2_1))

Response: Calculates Poisson LL of two points p1 and p2.


Instruction:
How would you implement a function in Python 3 that
calculates the Poisson distance between two vectors.

Response:
def poisson_dist(p1, p2):
    """
    Calculates the Poisson distance between two vectors.

    p1 can be a sparse matrix, while p2 has to be a dense matrix.
    """
    # ugh...
    p1_ = p1 + eps
    p2_ = p2 + eps
    return np.dot(p1_-p2_, np.log(p1_/p2_))


Instruction:
How would you explain what the following Python 3 function does
def delete(ctx, family_file, family_type, case_id):
    if not (family_file or case_id):
        LOG.error("Please provide a family file")
        ctx.abort()

    adapter = ctx.obj['adapter']

    # Get a ped_parser.Family object from family file
    family = None
    family_id = None
    if family_file:
        with open(family_file, 'r') as family_lines:
            family = get_case(
                family_lines=family_lines,
                family_type=family_type
            )
            family_id = family.family_id

    # There has to be a case_id or a family at this stage.
    case_id = case_id or family_id

    if not case_id:
        LOG.warning("Please provide a case id")
        ctx.abort()

    existing_case = adapter.case({'case_id': case_id})
    if not existing_case:
        LOG.warning("Case %s does not exist in database" %case_id)
        context.abort

    start_deleting = datetime.now()
    try:
        delete_command(
            adapter=adapter,
            case_obj=existing_case,
        )
    except (CaseError, IOError) as error:
        LOG.warning(error)
        ctx.abort()

Response: Delete the variants of a case.


Instruction:
How would you explain what the following Python 3 function does
def update_energy(self, bypass_check: bool = False):
        if bypass_check or (not bypass_check and self.update_time_check):
            self.get_weekly_energy()
            if 'week' in self.energy:
                self.get_monthly_energy()
                self.get_yearly_energy()
            if not bypass_check:
                self.update_energy_ts = time.time()

Response: Builds weekly monthly and yearly dictionaries


Instruction:
Here you have a function in Python 3, explain what it does
def main (args):
  if len(args) != 3:
    print(main.__doc__)
    sys.exit(1)

  d = libsedml.readSedML(args[1]);
  if ( d.getErrorLog().getNumFailsWithSeverity(libsedml.LIBSEDML_SEV_ERROR) > 0):
    print (d.getErrorLog().toString());
  else:
    libsedml.writeSedML(d, args[2]);

  return 0;

Response: A simple wrapper for the echo_sedml command.


Instruction:
How would you implement a function in Python 3 that
creates the aiohttp session and run the example.

Response:
async def main() -> None:
    """Create the aiohttp session and run the example."""
    logging.basicConfig(level=logging.INFO)

    async with ClientSession() as websession:
        try:
            client = Client(websession)

            await client.profile.login('<EMAIL>', '<PASSWORD>')
            _LOGGER.info('Account ID: %s', client.profile.account_id)

            summary = await client.profile.summary()
            _LOGGER.info('Account Summary: %s', summary)

            packages = await client.profile.packages()
            _LOGGER.info('Package Summary: %s', packages)
        except SeventeenTrackError as err:
            print(err)


Instruction:
Can you tell what is the following Python 3 function doing
def like_button_js_tag(context):
    if FACEBOOK_APP_ID is None:
        log.warning("FACEBOOK_APP_ID isn't setup correctly in your settings")

    # make sure FACEBOOK_APP_ID is setup correct and user is authenticated
    if FACEBOOK_APP_ID:

        request = context.get('request', None)
        if request:
            return {"LIKE_BUTTON_IS_VALID": True,
                    "facebook_app_id": FACEBOOK_APP_ID,
                    "channel_base_url": request.get_host()}

    # if it is here, it isn't a valid setup, return False to not show the tag.
    return {"LIKE_BUTTON_IS_VALID": False}

Response: This function will check if the user has set FACEBOOK_LIKE_APP_ID correctly in the django settings and if so will return True and the intercom_tag template to be displayed.


Instruction:
Can you tell what is the following Python 3 function doing
def like_button_tag(context):

    if FACEBOOK_APP_ID is None:
        log.warning("FACEBOOK_APP_ID isn't setup correctly in your settings")

    # make sure INTERCOM_APPID is setup correct and user is authenticated
    if FACEBOOK_APP_ID:

        request = context.get('request', None)
        if request:
            path_to_like = (
                "http://" + request.get_host() + request.get_full_path())

            show_send = true_false_converter(FACEBOOK_SHOW_SEND)
            like_width = FACEBOOK_LIKE_WIDTH
            show_faces = true_false_converter(FACEBOOK_SHOW_FACES)
            font = FACEBOOK_FONT

            return {"LIKE_BUTTON_IS_VALID": True,
                    "path_to_like": path_to_like,
                    "show_send": show_send,
                    "like_width": like_width,
                    "show_faces": show_faces,
                    "font": font,
                    "like_layout": FACEBOOK_LIKE_LAYOUT}

    # if it is here, it isn't a valid setup, return False to not show the tag.
    return {"LIKE_BUTTON_IS_VALID": False}

Response:
This function will check if the user has set the FACEBOOK_APP_ID and if so will check if they have the FACEBOOK_APP_ID setup correctly in the django settings and then will pass the data
        along to the intercom_tag template to be displayed.


Instruction:
How would you code a function in Python 3 to
add a single variant to the structural variants collection

Response:
def add_structural_variant(self, variant, max_window = 3000):
        """Add a variant to the structural variants collection

        The process of adding an SV variant differs quite a bit from the
        more straight forward case of SNV/INDEL.

        Variants are represented in the database by clusters that are two intervals,
        one interval for start(pos) and one for end. The size of the intervals changes
        according to the size of the variants. The maximum window size is a parameter.

        Here we need to search if the variant matches any of the existing
        clusters. Then we need to choose the closest cluster and update
        the boundaries for that cluster.

        Args:
            variant (dict): A variant dictionary
            max_window(int): Specify the maximum window size for large svs

                                        -
                                      -   -
                                    -       -
                                  -           -
                                -               -
                              -                   -
                           |-----|              |-----|
                          /\                         /\
                          |                          |
                pos - interval_size             end + interval_size
        """
        # This will return the cluster most similar to variant or None
        cluster = self.get_structural_variant(variant)
        # If there was no matcing cluster we need to create a new cluster
        if cluster is None:
            # The cluster will be populated with information later.
            cluster = {
                'chrom': variant['chrom'],
                'end_chrom': variant['end_chrom'],
                'sv_type': variant['sv_type'],
                'pos_sum': 0,
                'end_sum': 0,
                'observations': 0,
                'length': 0,
                'sv_type': variant['sv_type'],
                'families': [],

            }
            # Insert variant to get a _id
            _id = self.db.structural_variant.insert_one(cluster).inserted_id

            cluster['_id'] = _id

        case_id = variant.get('case_id')
        if case_id:
            # If the variant is already added for this case we continue
            # One case will only give duplicated information
            if case_id in cluster['families']:
                return
            else:
                # Insert the new case in the beginning of array
                cluster['families'].insert(0,case_id)
                # Make sure array does not grow out of bounds
                cluster['families'] = cluster['families'][:50]

        # Update number of times we have seen the event
        nr_events = cluster['observations'] + 1

        #                             -
        #                           -   -
        #                         -       -
        #                       -           -
        #                     -               -
        #                   -                   -
        #                |--.--|              |--.--|

        # This indicates the center for each of the end points for the event AFTER the new variant
        # is added to the cluster
        # i.e. the dots in the picture above
        pos_mean = int((cluster['pos_sum'] + variant['pos']) // (nr_events))
        end_mean = int((cluster['end_sum'] + variant['end']) // (nr_events))

        # We need to calculate the new cluster length
        # Handle translocation as a special case
        if cluster['sv_type'] != 'BND':
            cluster_len = end_mean - pos_mean
            # We need to adapt the interval size depending on the size of the cluster
            divider = 10
            if cluster_len < 1000:
                # We allow intervals for smaller variants to be relatively larger
                divider = 2
            elif cluster_len < 1000:
                # We allow intervals for smaller variants to be relatively larger
                divider = 5
            interval_size = int(min(round(cluster_len/divider, -2), max_window))
        else:
            # We need to treat translocations as a special case.
            # Set length to a huge number that mongodb can handle, float('inf') would not work.
            cluster_len = 10e10
            # This number seems large, if compared with SV size it is fairly small.
            interval_size = max_window * 2

        # If the length of SV is shorter than 500 the variant
        # is considered precise
        # Otherwise the interval size is closest whole 100 number
        res = self.db.structural_variant.find_one_and_update(
            {'_id': cluster['_id']},
            {
                '$inc': {
                    'observations': 1,
                    'pos_sum': variant['pos'],
                    'end_sum': variant['end'],
                },

                '$set': {
                    'pos_left': max(pos_mean - interval_size, 0),
                    'pos_right': pos_mean + interval_size,
                    'end_left': max(end_mean - interval_size, 0),
                    'end_right': end_mean + interval_size,
                    'families': cluster['families'],
                    'length': cluster_len,
                }
            }
        )

        # Insert an identity object to link cases to variants and clusters
        identity_obj = Identity(cluster_id=cluster['_id'], variant_id=variant['id_column'],
                                case_id=case_id)
        self.db.identity.insert_one(identity_obj)

        return


Instruction:
How would you code a function in Python 3 to
return the structural variant that is found in the database

Response:
def get_structural_variant(self, variant):
        """Check if there are any overlapping sv clusters

       Search the sv variants with chrom start end_chrom end and sv_type

       Args:
           variant (dict): A variant dictionary

       Returns:
           variant (dict): A variant dictionary
        """
        # Create a query for the database
        # This will include more variants than we want
        # The rest of the calculations will be done in python
        query = {
                'chrom': variant['chrom'],
                'end_chrom': variant['end_chrom'],
                'sv_type': variant['sv_type'],
                '$and': [
                    {'pos_left': {'$lte': variant['pos']}},
                    {'pos_right': {'$gte': variant['pos']}},
                ]
            }

        res = self.db.structural_variant.find(query).sort('pos_left',1)
        match = None
        distance = None
        closest_hit = None
        # First we check that the coordinates are correct
        # Then we count the distance to mean on both ends to see which variant is closest
        for hit in res:
            # We know from the query that the variants position is larger than the left most part of
            # the cluster.
            # If the right most part of the cluster is smaller than the variant position they do
            # not overlap
            if hit['end_left'] > variant['end']:
                continue
            if hit['end_right'] < variant['end']:
                continue

            # We need to calculate the distance to see what cluster that was closest to the variant
            distance = (abs(variant['pos'] - (hit['pos_left'] + hit['pos_right'])/2) +
                        abs(variant['end'] - (hit['end_left'] + hit['end_right'])/2))

            # If we have no cluster yet we set the curent to be the hit
            if closest_hit is None:
                match = hit
                closest_hit = distance
                continue

            # If the distance is closer than previous we choose current cluster
            if distance < closest_hit:
                # Set match to the current closest hit
                match = hit
                # Update the closest distance
                closest_hit = distance

        return match


Instruction:
Implement a function in Python 3 to
return all structural variants in the database

Response:
def get_sv_variants(self, chromosome=None, end_chromosome=None, sv_type=None,
                        pos=None, end=None):
        """Return all structural variants in the database

        Args:
            chromosome (str)
            end_chromosome (str)
            sv_type (str)
            pos (int): Left position of SV
            end (int): Right position of SV

        Returns:
            variants (Iterable(Variant))
        """
        query = {}

        if chromosome:
            query['chrom'] = chromosome
        if end_chromosome:
            query['end_chrom'] = end_chromosome
        if sv_type:
            query['sv_type'] = sv_type
        if pos:
            if not '$and' in query:
                query['$and'] = []
            query['$and'].append({'pos_left': {'$lte': pos}})
            query['$and'].append({'pos_right': {'$gte': pos}})

        if end:
            if not '$and' in query:
                query['$and'] = []
            query['$and'].append({'end_left': {'$lte': end}})
            query['$and'].append({'end_right': {'$gte': end}})

        LOG.info("Find all sv variants {}".format(query))

        return self.db.structural_variant.find(query).sort([('chrom', ASCENDING), ('pos_left', ASCENDING)])


Instruction:
Implement a function in Python 3 to
search what clusters a variant belongs to

Response:
def get_clusters(self, variant_id):
        """Search what clusters a variant belongs to

        Args:
            variant_id(str): From ID column in vcf

        Returns:
            clusters()
        """
        query = {'variant_id':variant_id}
        identities = self.db.identity.find(query)
        return identities


Instruction:
Explain what the following Python 3 code does
def run_state_estimation(data, clusters, dist='Poiss', reps=1, **kwargs):
    clusters = int(clusters)
    func = poisson_estimate_state
    dist = dist.lower()
    if dist=='poiss' or dist=='poisson':
        pass
    elif dist=='nb':
        func = nb_estimate_state
    elif dist=='zip':
        func = zip_estimate_state
    elif dist=='lognorm' or dist=='log-normal' or dist=='lognormal':
        func = log_norm_nmf
    elif dist=='gaussian' or dist=='norm' or dist=='normal':
        func = norm_nmf
    else:
        print('dist should be one of Poiss, NB, ZIP, LogNorm, or Gaussian. Using Poiss.')
    # TODO: estimate number of clusters
    if clusters == 0:
        from .gap_score import run_gap_k_selection, preproc_data
        data_tsvd = preproc_data(data, gene_subset=False)
        max_k, gap_vals, sk_vals = run_gap_k_selection(data_tsvd,
                k_min=1, k_max=50, skip=5, B=6)
        clusters = min(max_k, data.shape[0] - 1, data.shape[1] - 1)
    best_ll = np.inf
    best_M = None
    best_W = None
    for i in range(reps):
        results = func(data, clusters, **kwargs)
        M = results[0]
        W = results[1]
        if dist=='NB':
            ll = results[3]
        else:
            ll = results[2]
        if ll < best_ll:
            best_ll = ll
            best_M = M
            best_W = W
    return best_M, best_W, best_ll

Response: Runs state estimation for multiple initializations returning the result with the highest log - likelihood.


Instruction:
How would you explain what the following Python 3 function does
def turn_on(self):
        if self.device_status != 'on':
            body = helpers.req_body(self.manager, 'devicestatus')
            body['uuid'] = self.uuid
            body['status'] = 'on'
            head = helpers.req_headers(self.manager)

            r, _ = helpers.call_api('/131airPurifier/v1/device/deviceStatus',
                                    'put', json=body, headers=head)

            if r is not None and helpers.check_response(r, 'airpur_status'):
                self.device_status = 'on'
                return True
            else:
                return False

Response: Turn Air Purifier on


Instruction:
Implement a function in Python 3 to
adjust the fan speed by Specifying 1 2 3 as argument or cycle through speeds increasing by one

Response:
def fan_speed(self, speed: int = None) -> bool:
        """Adjust Fan Speed by Specifying 1,2,3 as argument or cycle
            through speeds increasing by one"""
        body = helpers.req_body(self.manager, 'devicestatus')
        body['uuid'] = self.uuid
        head = helpers.req_headers(self.manager)
        if self.details.get('mode') != 'manual':
            self.mode_toggle('manual')
        else:
            if speed is not None:
                level = int(self.details.get('level'))
                if speed == level:
                    return False
                elif speed in [1, 2, 3]:
                    body['level'] = speed
            else:
                if (level + 1) > 3:
                    body['level'] = 1
                else:
                    body['level'] = int(level + 1)

            r, _ = helpers.call_api('/131airPurifier/v1/device/updateSpeed',
                                    'put', json=body, headers=head)

            if r is not None and helpers.check_response(r, 'airpur_status'):
                self.details['level'] = body['level']
                return True
            else:
                return False


Instruction:
Make a summary of the following Python 3 code
def mode_toggle(self, mode: str) -> bool:
        head = helpers.req_headers(self.manager)
        body = helpers.req_body(self.manager, 'devicestatus')
        body['uuid'] = self.uuid
        if mode != body['mode'] and mode in ['sleep', 'auto', 'manual']:
            body['mode'] = mode
            if mode == 'manual':
                body['level'] = 1

            r, _ = helpers.call_api('/131airPurifier/v1/device/updateMode',
                                    'put', json=body, headers=head)

            if r is not None and helpers.check_response(r, 'airpur_status'):
                self.details['mode'] = mode
                return True

        return False

Response: Set mode to manual auto or sleep


Instruction:
Here you have a function in Python 3, explain what it does
def fourier_series(x, *a):
    output = 0
    output += a[0]/2
    w = a[1]
    for n in range(2, len(a), 2):
        n_ = n/2
        val1 = a[n]
        val2 = a[n+1]
        output += val1*np.sin(n_*x*w)
        output += val2*np.cos(n_*x*w)
    return output

Response: Compute the number of fourier series of a given interval x.


Instruction:
Create a Python 3 function to
calculate the distances along a graph from a start node to a new node.

Response:
def graph_distances(start, edges, distances):
    """
    Given an undirected adjacency list and a pairwise distance matrix between
    all nodes: calculates distances along graph from start node.

    Args:
        start (int): start node
        edges (list): adjacency list of tuples
        distances (array): 2d array of distances between nodes

    Returns:
        dict of node to distance from start
    """
    # convert adjacency list to adjacency dict
    adj = {x: [] for x in range(len(distances))}
    for n1, n2 in edges:
        adj[n1].append(n2)
        adj[n2].append(n1)
    # run dijkstra's algorithm
    to_visit = []
    new_dist = {}
    for n in adj[start]:
        heapq.heappush(to_visit, (distances[start, n], n))
    while to_visit:
        d, next_node = heapq.heappop(to_visit)
        if next_node not in new_dist:
            new_dist[next_node] = d
        for n in adj[next_node]:
            if n not in new_dist:
                heapq.heappush(to_visit, (d + distances[next_node, n], n))
    return new_dist


Instruction:
Create a Python 3 function for
returning a 1d array containing the pseudotime value of each cell in the tree.

Response:
def pseudotime(starting_node, edges, fitted_vals):
    """
    Args:
        starting_node (int): index of the starting node
        edges (list): list of tuples (node1, node2)
        fitted_vals (array): output of lineage (2 x cells)

    Returns:
        A 1d array containing the pseudotime value of each cell.
    """
    # TODO
    # 1. calculate a distance matrix...
    distances = np.array([[sum((x - y)**2) for x in fitted_vals.T] for y in fitted_vals.T])
    # 2. start from the root node/cell, calculate distance along graph
    distance_dict = graph_distances(starting_node, edges, distances)
    output = []
    for i in range(fitted_vals.shape[1]):
        output.append(distance_dict[i])
    return np.array(output)