Instruction:
Write a Python 3 function that can
build a new transcript object from a parsed transcript information.

Response:
def build_transcript(transcript, build='37'):
    """Build a transcript object

    These represents the transcripts that are parsed from the VCF, not
    the transcript definitions that are collected from ensembl.

        Args:
            transcript(dict): Parsed transcript information

        Returns:
            transcript_obj(dict)
    """

    # Transcripts has to have an id
    transcript_id = transcript['transcript_id']
    transcript_obj = dict(
        transcript_id = transcript_id
    )

    # Transcripts has to belong to a gene
    transcript_obj['hgnc_id'] = transcript['hgnc_id']

    if transcript.get('protein_id'):
        transcript_obj['protein_id'] = transcript['protein_id']

    if transcript.get('sift_prediction'):
        transcript_obj['sift_prediction'] = transcript['sift_prediction']

    if transcript.get('polyphen_prediction'):
        transcript_obj['polyphen_prediction'] = transcript['polyphen_prediction']

    if transcript.get('swiss_prot'):
        transcript_obj['swiss_prot'] = transcript['swiss_prot']

    if transcript.get('pfam_domain'):
        transcript_obj['pfam_domain'] = transcript.get('pfam_domain')

    if transcript.get('prosite_profile'):
        transcript_obj['prosite_profile'] = transcript.get('prosite_profile')

    if transcript.get('smart_domain'):
        transcript_obj['smart_domain'] = transcript.get('smart_domain')

    if transcript.get('biotype'):
        transcript_obj['biotype'] = transcript.get('biotype')

    if transcript.get('functional_annotations'):
        transcript_obj['functional_annotations'] = transcript['functional_annotations']

    if transcript.get('region_annotations'):
        transcript_obj['region_annotations'] = transcript['region_annotations']

    if transcript.get('exon'):
        transcript_obj['exon'] = transcript.get('exon')

    if transcript.get('intron'):
        transcript_obj['intron'] = transcript.get('intron')

    if transcript.get('strand'):
        transcript_obj['strand'] = transcript.get('strand')

    if transcript.get('coding_sequence_name'):
        transcript_obj['coding_sequence_name'] = transcript['coding_sequence_name']

    if transcript.get('protein_sequence_name'):
        transcript_obj['protein_sequence_name'] = transcript['protein_sequence_name']

    transcript_obj['is_canonical'] = transcript.get('is_canonical', False)

    return transcript_obj


Instruction:
Implement a Python 3 function for
updating an existing user.

Response:
def update_user(self, user_obj):
        """Update an existing user.

            Args:
                user_obj(dict)

            Returns:
                updated_user(dict)
        """
        LOG.info("Updating user %s", user_obj['_id'])
        updated_user = self.user_collection.find_one_and_replace(
            {'_id': user_obj['_id']},
            user_obj,
            return_document=pymongo.ReturnDocument.AFTER
        )
        return updated_user


Instruction:
Can you implement a function in Python 3 that
adds a user object to the database.

Response:
def add_user(self, user_obj):
        """Add a user object to the database

            Args:
                user_obj(scout.models.User): A dictionary with user information

            Returns:
                user_info(dict): a copy of what was inserted
        """
        LOG.info("Adding user %s to the database", user_obj['email'])
        if not '_id' in user_obj:
            user_obj['_id'] = user_obj['email']

        try:
            self.user_collection.insert_one(user_obj)
            LOG.debug("User inserted")
        except DuplicateKeyError as err:
            raise IntegrityError("User {} already exists in database".format(user_obj['email']))

        return user_obj


Instruction:
Given the following Python 3 function, write the documentation
def users(self, institute=None):
        query = {}
        if institute:
            LOG.info("Fetching all users from institute %s", institute)
            query = {'institutes': {'$in': [institute]}}
        else:
            LOG.info("Fetching all users")

        res = self.user_collection.find(query)
        return res

Response: Returns all users from the database


Instruction:
Can you tell what is the following Python 3 function doing
def user(self, email):
        LOG.info("Fetching user %s", email)
        user_obj = self.user_collection.find_one({'_id': email})

        return user_obj

Response: Fetch a user from the database.


Instruction:
Can you create a Python 3 function that
deletes a user from the database

Response:
def delete_user(self, email):
        """Delete a user from the database

        Args:
            email(str)

        Returns:
            user_obj(dict)

        """
        LOG.info("Deleting user %s", email)
        user_obj = self.user_collection.delete_one({'_id': email})

        return user_obj


Instruction:
Explain what the following Python 3 code does
def build_compound(compound):

    compound_obj = dict(
        variant = compound['variant'],
        display_name = compound['display_name'],
        combined_score = float(compound['score'])
    )

    return compound_obj

Response: Builds a compound object from a dictionary.


Instruction:
How would you explain what the following Python 3 function does
def remote_static():
    file_path = request.args.get('file')

    range_header = request.headers.get('Range', None)
    if not range_header and file_path.endswith('.bam'):
        return abort(500)

    new_resp = send_file_partial(file_path)
    return new_resp

Response: Stream large static files with special requirements.


Instruction:
Here you have a function in Python 3, explain what it does
def igv():
    chrom = request.args.get('contig')
    if chrom == 'MT':
        chrom = 'M'

    start = request.args.get('start')
    stop = request.args.get('stop')

    locus = "chr{0}:{1}-{2}".format(chrom,start,stop)
    LOG.debug('Displaying locus %s', locus)

    chromosome_build = request.args.get('build')
    LOG.debug('Chromosome build is %s', chromosome_build)

    samples = request.args.getlist('sample')
    bam_files = request.args.getlist('bam')
    bai_files = request.args.getlist('bai')
    LOG.debug('loading the following tracks: %s', bam_files)

    display_obj={}

    # Add chromosome build info to the track object
    fastaURL = ''
    indexURL = ''
    cytobandURL = ''
    gene_track_format = ''
    gene_track_URL = ''
    gene_track_indexURL = ''

    if chromosome_build == "GRCh38" or chrom == 'M':
        fastaURL = 'https://s3.amazonaws.com/igv.broadinstitute.org/genomes/seq/hg38/hg38.fa'
        indexURL = 'https://s3.amazonaws.com/igv.broadinstitute.org/genomes/seq/hg38/hg38.fa.fai'
        cytobandURL = 'https://s3.amazonaws.com/igv.broadinstitute.org/annotations/hg38/cytoBandIdeo.txt'
        gene_track_format = 'gtf'
        gene_track_URL = 'https://s3.amazonaws.com/igv.broadinstitute.org/annotations/hg38/genes/Homo_sapiens.GRCh38.80.sorted.gtf.gz'
        gene_track_indexURL = 'https://s3.amazonaws.com/igv.broadinstitute.org/annotations/hg38/genes/Homo_sapiens.GRCh38.80.sorted.gtf.gz.tbi'

    else:
        fastaURL = 'https://s3.amazonaws.com/igv.broadinstitute.org/genomes/seq/hg19/hg19.fasta'
        indexURL = 'https://s3.amazonaws.com/igv.broadinstitute.org/genomes/seq/hg19/hg19.fasta.fai'
        cytobandURL = 'https://s3.amazonaws.com/igv.broadinstitute.org/genomes/seq/hg19/cytoBand.txt'
        gene_track_format = 'bed'
        gene_track_URL = 'https://s3.amazonaws.com/igv.broadinstitute.org/annotations/hg19/genes/refGene.hg19.bed.gz'
        gene_track_indexURL = 'https://s3.amazonaws.com/igv.broadinstitute.org/annotations/hg19/genes/refGene.hg19.bed.gz.tbi'

    display_obj['reference_track'] = {
        'fastaURL' : fastaURL,
        'indexURL' : indexURL,
        'cytobandURL' : cytobandURL
    }

    display_obj['genes_track'] = {
        'name' : 'Genes',
        'type' : 'annotation',
        'format': gene_track_format,
        'sourceType': 'file',
        'url' : gene_track_URL,
        'indexURL' : gene_track_indexURL,
        'displayMode' : 'EXPANDED'
    }

    sample_tracks = []
    counter = 0
    for sample in samples:
        # some samples might not have an associated bam file, take care if this
        if bam_files[counter]:
            sample_tracks.append({ 'name' : sample, 'url' : bam_files[counter],
                                   'indexURL' : bai_files[counter],
                                   'height' : 700, 'maxHeight' : 2000})
        counter += 1

    display_obj['sample_tracks'] = sample_tracks

    if request.args.get('center_guide'):
        display_obj['display_center_guide'] = True
    else:
        display_obj['display_center_guide'] = False

    return render_template('alignviewers/igv_viewer.html', locus=locus, **display_obj )

Response: Visualize BAM alignments using igv. js.


Instruction:
Write a Python 3 script to
build a disease term object from a dictionary with phenotype information.

Response:
def build_disease_term(disease_info, alias_genes={}):
    """Build a disease phenotype object

    Args:
        disease_info(dict): Dictionary with phenotype information
        alias_genes(dict): {
                    <alias_symbol>: {
                                        'true': hgnc_id or None,
                                        'ids': [<hgnc_id>, ...]}}

    Returns:
        disease_obj(dict): Formated for mongodb

        disease_term = dict(
            _id = str, # Same as disease_id
            disease_id = str, # required, like OMIM:600233
            disase_nr = int, # The disease nr, required
            description = str, # required
            source = str, # required
            inheritance = list, # list of strings
            genes = list, # List with integers that are hgnc_ids
            hpo_terms = list, # List with str that are hpo_terms
        )

    """

    try:
        disease_nr = int(disease_info['mim_number'])
    except KeyError:
        raise KeyError("Diseases has to have a disease number")
    except ValueError:
        raise KeyError("Diseases nr has to be integer")

    disease_id = "{0}:{1}".format('OMIM', disease_nr)

    LOG.debug("Building disease term %s", disease_id)

    try:
        description = disease_info['description']
    except KeyError:
        raise KeyError("Diseases has to have a description")

    disease_obj = DiseaseTerm(
        disease_id=disease_id,
        disease_nr=disease_nr,
        description=description,
        source='OMIM'
    )

    # Check if there where any inheritance information
    inheritance_models = disease_info.get('inheritance')
    if inheritance_models:
        disease_obj['inheritance'] = list(inheritance_models)

    hgnc_ids = set()
    for hgnc_symbol in disease_info.get('hgnc_symbols', []):
        ## TODO need to consider genome build here?
        if hgnc_symbol in alias_genes:
            # If the symbol identifies a unique gene we add that
            if alias_genes[hgnc_symbol]['true']:
                hgnc_ids.add(alias_genes[hgnc_symbol]['true'])
            else:
                for hgnc_id in alias_genes[hgnc_symbol]['ids']:
                    hgnc_ids.add(hgnc_id)
        else:
            LOG.debug("Gene symbol %s could not be found in database", hgnc_symbol)

    disease_obj['genes'] = list(hgnc_ids)

    if 'hpo_terms' in disease_info:
        disease_obj['hpo_terms'] = list(disease_info['hpo_terms'])

    return disease_obj


Instruction:
Can you generate a brief explanation for the following Python 3 code
def load_exons(adapter, exon_lines, build='37', ensembl_genes=None):
    # Fetch all genes with ensemblid as keys
    ensembl_genes = ensembl_genes or adapter.ensembl_genes(build)
    hgnc_id_transcripts = adapter.id_transcripts_by_gene(build=build)

    if isinstance(exon_lines, DataFrame):
        exons = parse_ensembl_exon_request(exon_lines)
        nr_exons = exon_lines.shape[0]
    else:
        exons = parse_ensembl_exons(exon_lines)
        nr_exons = 1000000

    start_insertion = datetime.now()
    loaded_exons = 0
    LOG.info("Loading exons...")
    with progressbar(exons, label="Loading exons", length=nr_exons) as bar:
        for exon in bar:
            ensg_id = exon['gene']
            enst_id = exon['transcript']
            gene_obj = ensembl_genes.get(ensg_id)

            if not gene_obj:
                continue

            hgnc_id = gene_obj['hgnc_id']

            if not enst_id in hgnc_id_transcripts[hgnc_id]:
                continue

            exon['hgnc_id'] = hgnc_id

            exon_obj = build_exon(exon, build)
            adapter.load_exon(exon_obj)
            loaded_exons += 1

    LOG.info('Number of exons in build {0}: {1}'.format(build, nr_exons))
    LOG.info('Number loaded: {0}'.format(loaded_exons))
    LOG.info('Time to load exons: {0}'.format(datetime.now() - start_insertion))

Response: Load all the exons from the given list of lines into the database.


Instruction:
Write a Python 3 function for
parsing a single variant and return a dictionary with the parsed variant information.

Response:
def parse_variant(variant, case, variant_type='clinical',
                 rank_results_header=None, vep_header=None,
                 individual_positions=None, category=None):
    """Return a parsed variant

        Get all the necessary information to build a variant object

    Args:
        variant(cyvcf2.Variant)
        case(dict)
        variant_type(str): 'clinical' or 'research'
        rank_results_header(list)
        vep_header(list)
        individual_positions(dict): Explain what position each individual has
                                    in vcf
        category(str): 'snv', 'sv', 'str' or 'cancer'

    Returns:
        parsed_variant(dict): Parsed variant
    """
    # These are to display how the rank score is built
    rank_results_header = rank_results_header or []
    # Vep information
    vep_header = vep_header or []

    parsed_variant = {}

    # Create the ID for the variant
    case_id = case['_id']
    if '-' in case_id:
        logger.debug('internal case id detected')
        genmod_key = case['display_name']
    else:
        genmod_key = case['_id']

    chrom_match = CHR_PATTERN.match(variant.CHROM)
    chrom = chrom_match.group(2)

    # Builds a dictionary with the different ids that are used

    if variant.ALT:
        alt=variant.ALT[0]
    elif not variant.ALT and category == "str":
        alt='.'

    parsed_variant['ids'] = parse_ids(
        chrom=chrom,
        pos=variant.POS,
        ref=variant.REF,
        alt=alt,
        case_id=case_id,
        variant_type=variant_type,
    )
    parsed_variant['case_id'] = case_id
    # type can be 'clinical' or 'research'
    parsed_variant['variant_type'] = variant_type
    # category is sv or snv
    # cyvcf2 knows if it is a sv, indel or snv variant
    if not category:
        category = variant.var_type
        if category == 'indel':
            category = 'snv'
        if category == 'snp':
            category = 'snv'

    parsed_variant['category'] = category

    ################# General information #################

    parsed_variant['reference'] = variant.REF

    ### We allways assume splitted and normalized vcfs!!!
    if len(variant.ALT) > 1:
        raise VcfError("Variants are only allowed to have one alternative")
    parsed_variant['alternative'] = alt

    # cyvcf2 will set QUAL to None if '.' in vcf
    parsed_variant['quality'] = variant.QUAL

    if variant.FILTER:
        parsed_variant['filters'] = variant.FILTER.split(';')
    else:
        parsed_variant['filters'] = ['PASS']

    # Add the dbsnp ids
    parsed_variant['dbsnp_id'] = variant.ID

    # This is the id of other position in translocations
    # (only for specific svs)
    parsed_variant['mate_id'] = None

    ################# Position specific #################
    parsed_variant['chromosome'] = chrom

    coordinates = parse_coordinates(variant, category)

    parsed_variant['position'] = coordinates['position']
    parsed_variant['sub_category'] = coordinates['sub_category']
    parsed_variant['mate_id'] = coordinates['mate_id']
    parsed_variant['end'] = coordinates['end']
    parsed_variant['length'] = coordinates['length']
    parsed_variant['end_chrom'] = coordinates['end_chrom']
    parsed_variant['cytoband_start'] = coordinates['cytoband_start']
    parsed_variant['cytoband_end'] = coordinates['cytoband_end']

    ################# Add rank score #################
    # The rank score is central for displaying variants in scout.

    rank_score = parse_rank_score(variant.INFO.get('RankScore', ''), genmod_key)
    parsed_variant['rank_score'] = rank_score or 0

    ################# Add gt calls #################
    if individual_positions and case['individuals']:
        parsed_variant['samples'] = parse_genotypes(variant, case['individuals'],
                                                    individual_positions)
    else:
        parsed_variant['samples'] = []

    ################# Add compound information #################
    compounds = parse_compounds(compound_info=variant.INFO.get('Compounds'),
                                case_id=genmod_key,
                                variant_type=variant_type)
    if compounds:
        parsed_variant['compounds'] = compounds

    ################# Add inheritance patterns #################

    genetic_models = parse_genetic_models(variant.INFO.get('GeneticModels'), genmod_key)
    if genetic_models:
        parsed_variant['genetic_models'] = genetic_models

    ################# Add autozygosity calls if present #################

    azlength = variant.INFO.get('AZLENGTH')
    if azlength:
        parsed_variant['azlength'] = int(azlength)

    azqual = variant.INFO.get('AZQUAL')
    if azqual:
        parsed_variant['azqual'] = float(azqual)

    ################ Add STR info if present ################

    # repeat id generally corresponds to gene symbol
    repeat_id = variant.INFO.get('REPID')
    if repeat_id:
        parsed_variant['str_repid'] = str(repeat_id)

    # repeat unit - used e g in PanelApp naming of STRs
    repeat_unit = variant.INFO.get('RU')
    if repeat_unit:
        parsed_variant['str_ru'] = str(repeat_unit)

    # repeat ref - reference copy number
    repeat_ref = variant.INFO.get('REF')
    if repeat_ref:
        parsed_variant['str_ref'] = int(repeat_ref)

    # repeat len - number of repeats found in case
    repeat_len = variant.INFO.get('RL')
    if repeat_len:
        parsed_variant['str_len'] = int(repeat_len)

    # str status - this indicates the severity of the expansion level
    str_status = variant.INFO.get('STR_STATUS')
    if str_status:
        parsed_variant['str_status'] = str(str_status)

    ################# Add gene and transcript information #################
    raw_transcripts = []
    if vep_header:
        vep_info = variant.INFO.get('CSQ')
        if vep_info:
            raw_transcripts = (dict(zip(vep_header, transcript_info.split('|')))
                               for transcript_info in vep_info.split(','))

    parsed_transcripts = []
    dbsnp_ids = set()
    cosmic_ids = set()
    for parsed_transcript in parse_transcripts(raw_transcripts, parsed_variant['alternative']):
        parsed_transcripts.append(parsed_transcript)
        for dbsnp in parsed_transcript.get('dbsnp', []):
            dbsnp_ids.add(dbsnp)
        for cosmic in parsed_transcript.get('cosmic', []):
            cosmic_ids.add(cosmic)

    # The COSMIC tag in INFO is added via VEP and/or bcftools annotate

    cosmic_tag = variant.INFO.get('COSMIC')
    if cosmic_tag:
        cosmic_ids.add(cosmic_tag[4:])

    if (dbsnp_ids and not parsed_variant['dbsnp_id']):
        parsed_variant['dbsnp_id'] = ';'.join(dbsnp_ids)

    if cosmic_ids:
        parsed_variant['cosmic_ids'] = list(cosmic_ids)

    gene_info = parse_genes(parsed_transcripts)

    parsed_variant['genes'] = gene_info

    hgnc_ids = set([])

    for gene in parsed_variant['genes']:
        hgnc_ids.add(gene['hgnc_id'])

    parsed_variant['hgnc_ids'] = list(hgnc_ids)

    ################# Add clinsig prediction #################
    if variant.INFO.get('CLNACC'):
        acc = variant.INFO.get('CLNACC')
    else:
        acc = variant.INFO.get('CLNVID')
    clnsig_predictions = parse_clnsig(
        acc=acc,
        sig=variant.INFO.get('CLNSIG'),
        revstat=variant.INFO.get('CLNREVSTAT'),
        transcripts=parsed_transcripts
        )

    if clnsig_predictions:
        parsed_variant['clnsig'] = clnsig_predictions

    ################# Add the frequencies #################
    frequencies = parse_frequencies(variant, parsed_transcripts)

    parsed_variant['frequencies'] = frequencies

    # parse out old local observation count
    local_obs_old = variant.INFO.get('Obs')
    if local_obs_old:
        parsed_variant['local_obs_old'] = int(local_obs_old)

    local_obs_hom_old = variant.INFO.get('Hom')
    if local_obs_hom_old:
        parsed_variant['local_obs_hom_old'] = int(local_obs_hom_old)

    ###################### Add severity predictions ######################
    cadd = parse_cadd(variant, parsed_transcripts)
    if cadd:
        parsed_variant['cadd_score'] = cadd

    spidex = variant.INFO.get('SPIDEX')
    if spidex:
        parsed_variant['spidex'] = float(spidex)

    ###################### Add conservation ######################

    parsed_variant['conservation'] = parse_conservations(variant)

    parsed_variant['callers'] = parse_callers(variant, category=category)

    rank_result = variant.INFO.get('RankResult')
    if rank_result:
        results = [int(i) for i in rank_result.split('|')]
        parsed_variant['rank_result'] = dict(zip(rank_results_header, results))

    ###################### Add SV specific annotations ######################
    sv_frequencies = parse_sv_frequencies(variant)
    for key in sv_frequencies:
        parsed_variant['frequencies'][key] = sv_frequencies[key]

    ###################### Add Cancer specific annotations ######################
    # MSK_MVL indicates if variants are in the MSK managed variant list
    # https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5437632/
    mvl_tag = variant.INFO.get('MSK_MVL')
    if mvl_tag:
        parsed_variant['mvl_tag'] = True

    return parsed_variant


Instruction:
Create a Python 3 function to
update all compounds for a case

Response:
def compounds(context, case_id):
    """
    Update all compounds for a case
    """
    adapter = context.obj['adapter']
    LOG.info("Running scout update compounds")
    # Check if the case exists
    case_obj = adapter.case(case_id)

    if not case_obj:
        LOG.warning("Case %s could not be found", case_id)
        context.abort()

    try:
        adapter.update_case_compounds(case_obj)
    except Exception as err:
        LOG.warning(err)
        context.abort()


Instruction:
Write a Python 3 function that can
update a gene object with many links

Response:
def add_gene_links(gene_obj, build=37):
    """Update a gene object with links

    Args:
        gene_obj(dict)
        build(int)

    Returns:
        gene_obj(dict): gene_obj updated with many links
    """
    try:
        build = int(build)
    except ValueError:
        build = 37
    # Add links that use the hgnc_id
    hgnc_id = gene_obj['hgnc_id']

    gene_obj['hgnc_link'] = genenames(hgnc_id)
    gene_obj['omim_link'] = omim(hgnc_id)
    # Add links that use ensembl_id
    if not 'ensembl_id' in gene_obj:
        ensembl_id = gene_obj.get('common',{}).get('ensembl_id')
    else:
        ensembl_id = gene_obj['ensembl_id']
    ensembl_37_link = ensembl(ensembl_id, build=37)
    ensembl_38_link = ensembl(ensembl_id, build=38)
    gene_obj['ensembl_37_link'] = ensembl_37_link
    gene_obj['ensembl_38_link'] = ensembl_38_link
    gene_obj['ensembl_link'] = ensembl_37_link
    if build == 38:
        gene_obj['ensembl_link'] = ensembl_38_link
    gene_obj['hpa_link'] = hpa(ensembl_id)
    gene_obj['string_link'] = string(ensembl_id)
    gene_obj['reactome_link'] = reactome(ensembl_id)
    gene_obj['clingen_link'] = clingen(hgnc_id)
    gene_obj['expression_atlas_link'] = expression_atlas(ensembl_id)
    gene_obj['exac_link'] = exac(ensembl_id)
    # Add links that use entrez_id
    gene_obj['entrez_link'] = entrez(gene_obj.get('entrez_id'))
    # Add links that use omim id
    gene_obj['omim_link'] = omim(gene_obj.get('omim_id'))
    # Add links that use hgnc_symbol
    gene_obj['ppaint_link'] = ppaint(gene_obj['hgnc_symbol'])
    # Add links that use vega_id
    gene_obj['vega_link'] = vega(gene_obj.get('vega_id'))
    # Add links that use ucsc link
    gene_obj['ucsc_link'] = ucsc(gene_obj.get('ucsc_id'))


Instruction:
Create a Python 3 function for
querying the aliases and transcripts of a given hgnc symbol

Response:
def hgnc(ctx, hgnc_symbol, hgnc_id, build):
    """
    Query the hgnc aliases
    """
    adapter = ctx.obj['adapter']

    if not (hgnc_symbol or hgnc_id):
        log.warning("Please provide a hgnc symbol or hgnc id")
        ctx.abort()

    if hgnc_id:
        result = adapter.hgnc_gene(hgnc_id, build=build)
        if result:
            hgnc_symbol = result['hgnc_symbol']
        else:
            log.warning("Gene with id %s could not be found", hgnc_id)
            ctx.abort()

    result = adapter.hgnc_genes(hgnc_symbol, build=build)

    if result.count() == 0:
        log.info("No results found")

    else:
        click.echo("#hgnc_id\thgnc_symbol\taliases\ttranscripts")
        for gene in result:
            click.echo("{0}\t{1}\t{2}\t{3}".format(
                gene['hgnc_id'],
                gene['hgnc_symbol'],
                ', '.join(gene['aliases']),
                ', '.join(tx['ensembl_transcript_id'] for tx in gene['transcripts']),
            ))


Instruction:
Implement a function in Python 3 to
parse an hgnc formated line into a dict with the relevant info

Response:
def parse_hgnc_line(line, header):
    """Parse an hgnc formated line

        Args:
            line(list): A list with hgnc gene info
            header(list): A list with the header info

        Returns:
            hgnc_info(dict): A dictionary with the relevant info
    """
    hgnc_gene = {}
    line = line.rstrip().split('\t')
    raw_info = dict(zip(header, line))
    # Skip all genes that have status withdrawn
    if 'Withdrawn' in raw_info['status']:
        return hgnc_gene

    hgnc_symbol = raw_info['symbol']
    hgnc_gene['hgnc_symbol'] = hgnc_symbol
    hgnc_gene['hgnc_id'] = int(raw_info['hgnc_id'].split(':')[-1])
    hgnc_gene['description'] = raw_info['name']

    # We want to have the current symbol as an alias
    aliases = set([hgnc_symbol, hgnc_symbol.upper()])
    # We then need to add both the previous symbols and
    # alias symbols
    previous_names = raw_info['prev_symbol']
    if previous_names:
        for alias in previous_names.strip('"').split('|'):
            aliases.add(alias)

    alias_symbols = raw_info['alias_symbol']
    if alias_symbols:
        for alias in alias_symbols.strip('"').split('|'):
            aliases.add(alias)

    hgnc_gene['previous_symbols'] = list(aliases)

    # We need the ensembl_gene_id to link the genes with ensembl
    hgnc_gene['ensembl_gene_id'] = raw_info.get('ensembl_gene_id')

    omim_id = raw_info.get('omim_id')
    if omim_id:
        hgnc_gene['omim_id'] = int(omim_id.strip('"').split('|')[0])
    else:
        hgnc_gene['omim_id'] = None

    entrez_id = hgnc_gene['entrez_id'] = raw_info.get('entrez_id')
    if entrez_id:
        hgnc_gene['entrez_id'] = int(entrez_id)
    else:
        hgnc_gene['entrez_id'] = None

    # These are the primary transcripts according to HGNC
    ref_seq = raw_info.get('refseq_accession')
    if ref_seq:
        hgnc_gene['ref_seq'] = ref_seq.strip('"').split('|')
    else:
        hgnc_gene['ref_seq'] = []

    uniprot_ids = raw_info.get('uniprot_ids')
    if uniprot_ids:
        hgnc_gene['uniprot_ids'] = uniprot_ids.strip('""').split('|')
    else:
        hgnc_gene['uniprot_ids'] = []

    ucsc_id = raw_info.get('ucsc_id')
    if ucsc_id:
        hgnc_gene['ucsc_id'] = ucsc_id
    else:
        hgnc_gene['ucsc_id'] = None

    vega_id = raw_info.get('vega_id')
    if vega_id:
        hgnc_gene['vega_id'] = vega_id
    else:
        hgnc_gene['vega_id'] = None

    return hgnc_gene


Instruction:
Can you generate a brief explanation for the following Python 3 code
def parse_hgnc_genes(lines):
    header = []
    logger.info("Parsing hgnc genes...")
    for index, line in enumerate(lines):
        if index == 0:
            header = line.split('\t')
        elif len(line) > 1:
            hgnc_gene = parse_hgnc_line(line=line, header=header)
            if hgnc_gene:
                yield hgnc_gene

Response: Parse lines with HGNC formated genes


Instruction:
Make a summary of the following Python 3 code
def create_submission(self, user_id, institute_id):

        submission_obj = {
            'status' : 'open',
            'created_at' : datetime.now(),
            'user_id' : user_id,
            'institute_id' : institute_id
        }
        LOG.info("Creating a new clinvar submission for user '%s' and institute %s", user_id, institute_id)
        result = self.clinvar_submission_collection.insert_one(submission_obj)
        return result.inserted_id

Response: Create an open clinvar submission for a user and an institute ID.


Instruction:
How would you implement a function in Python 3 that
deletes a submission object along with all associated clinvar objects and all associated casedata objects.

Response:
def delete_submission(self, submission_id):
        """Deletes a Clinvar submission object, along with all associated clinvar objects (variants and casedata)

            Args:
                submission_id(str): the ID of the submission to be deleted

            Returns:
                deleted_objects(int): the number of associated objects removed (variants and/or casedata)
                deleted_submissions(int): 1 if it's deleted, 0 if something went wrong
        """
        LOG.info("Deleting clinvar submission %s", submission_id)
        submission_obj = self.clinvar_submission_collection.find_one({ '_id' : ObjectId(submission_id)})

        submission_variants = submission_obj.get('variant_data')
        submission_casedata = submission_obj.get('case_data')

        submission_objects = []

        if submission_variants and submission_casedata:
            submission_objects = submission_variants + submission_casedata
        elif submission_variants:
            submission_objects = submission_variants
        elif submission_casedata:
            submission_objects = submission_casedata

        # Delete all variants and casedata objects associated with this submission
        result = self.clinvar_collection.delete_many({'_id': { "$in": submission_objects} })
        deleted_objects = result.deleted_count

        # Delete the submission itself
        result = self.clinvar_submission_collection.delete_one({'_id': ObjectId(submission_id)})
        deleted_submissions = result.deleted_count

        #return deleted_count, deleted_submissions
        return deleted_objects,deleted_submissions


Instruction:
Write a Python 3 script to
retrieve the database id of an open clinvar submission for a user and institute. If none is available create one and return it.

Response:
def get_open_clinvar_submission(self, user_id, institute_id):
        """Retrieve the database id of an open clinvar submission for a user and institute,
           if none is available then create a new submission and return it

           Args:
                user_id(str): a user ID
                institute_id(str): an institute ID

           Returns:
                submission(obj) : an open clinvar submission object
        """

        LOG.info("Retrieving an open clinvar submission for user '%s' and institute %s", user_id, institute_id)
        query = dict(user_id=user_id, institute_id=institute_id, status='open')
        submission = self.clinvar_submission_collection.find_one(query)

        # If there is no open submission for this user and institute, create one
        if submission is None:
            submission_id = self.create_submission(user_id, institute_id)
            submission = self.clinvar_submission_collection.find_one({'_id':submission_id})

        return submission


Instruction:
Can you tell what is the following Python 3 function doing
def update_clinvar_id(self, clinvar_id, submission_id ):
        updated_submission = self.clinvar_submission_collection.find_one_and_update( {'_id': ObjectId(submission_id)}, { '$set' : {'clinvar_subm_id' : clinvar_id, 'updated_at': datetime.now()} }, upsert=True, return_document=pymongo.ReturnDocument.AFTER )
        return updated_submission

Response: updates the submission ID in a clinvar submission object


Instruction:
Can you write a function in Python 3 where it
returns the official Clinvar submission ID for a submission object

Response:
def get_clinvar_id(self, submission_id):
        """Returns the official Clinvar submission ID for a submission object

            Args:
                submission_id(str): submission_id(str) : id of the submission

            Returns:
                clinvar_subm_id(str): a string with a format: SUB[0-9]. It is obtained from clinvar portal when starting a new submission

        """
        submission_obj = self.clinvar_submission_collection.find_one({'_id': ObjectId(submission_id)})
        clinvar_subm_id = submission_obj.get('clinvar_subm_id') # This key does not exist if it was not previously provided by user
        return clinvar_subm_id


Instruction:
Can you create a Python 3 function that
adds the given objects to the submission and updates the coresponding submission object with their id ArcATATATATATATATATATATATATATATATATATATATATATATATATATATATATATATATATATATATATATATATATATATATATATATATATATATATATATATATATATATATATATATATATATATATATATATATATATATATATATAT

Response:
def add_to_submission(self, submission_id, submission_objects):
        """Adds submission_objects to clinvar collection and update the coresponding submission object with their id

            Args:
                submission_id(str) : id of the submission to be updated
                submission_objects(tuple): a tuple of 2 elements coresponding to a list of variants and a list of case data objects to add to submission

            Returns:
                updated_submission(obj): an open clinvar submission object, updated
        """

        LOG.info("Adding new variants and case data to clinvar submission '%s'", submission_id)

        # Insert variant submission_objects into clinvar collection
        # Loop over the objects
        for var_obj in submission_objects[0]:
            try:
                result = self.clinvar_collection.insert_one(var_obj)
                self.clinvar_submission_collection.update_one({'_id':submission_id}, {'$push': { 'variant_data' : str(result.inserted_id) }}, upsert=True)
            except pymongo.errors.DuplicateKeyError:
                LOG.error("Attepted to insert a clinvar variant which is already in DB!")

        # Insert casedata submission_objects into clinvar collection
        if submission_objects[1]:
            # Loop over the objects
            for case_obj in submission_objects[1]:
                try:
                    result = self.clinvar_collection.insert_one(case_obj)
                    self.clinvar_submission_collection.update_one({'_id':submission_id}, {'$push': { 'case_data': str(result.inserted_id)}}, upsert=True)
                except pymongo.errors.DuplicateKeyError:
                    LOG.error("One or more casedata object is already present in clinvar collection!")

        updated_submission = self.clinvar_submission_collection.find_one_and_update( {'_id':submission_id}, { '$set' : {'updated_at': datetime.now()} }, return_document=pymongo.ReturnDocument.AFTER )
        return updated_submission


Instruction:
Write a Python 3 script for
updating the status of a clinvar submission

Response:
def update_clinvar_submission_status(self, user_id, submission_id, status):
        """Set a clinvar submission ID to 'closed'

            Args:
                submission_id(str): the ID of the clinvar submission to close

            Return
                updated_submission(obj): the submission object with a 'closed' status

        """
        LOG.info('closing clinvar submission "%s"', submission_id)

        if status == 'open': # just close the submission its status does not affect the other submissions for this user
            # Close all other submissions for this user and then open the desired one
            self.clinvar_submission_collection.update_many(
                {'user_id' : user_id},
                {'$set' :
                    {'status' : 'closed', 'updated_at' : datetime.now()}
                }
            )
        updated_submission = self.clinvar_submission_collection.find_one_and_update(
            {'_id'  : ObjectId(submission_id)},
            {'$set' :
                {'status' : status, 'updated_at' : datetime.now()}
            },
            return_document=pymongo.ReturnDocument.AFTER
        )

        return updated_submission


Instruction:
Write a Python 3 function for
getting all open and closed clinvar submissions created by a user for an institute.

Response:
def clinvar_submissions(self, user_id, institute_id):
        """Collect all open and closed clinvar submission created by a user for an institute

            Args:
                user_id(str): a user ID
                institute_id(str): an institute ID

            Returns:
                submissions(list): a list of clinvar submission objects
        """
        LOG.info("Retrieving all clinvar submissions for user '%s', institute '%s'", user_id, institute_id)
        # get first all submission objects
        query = dict(user_id=user_id, institute_id=institute_id)
        results = list(self.clinvar_submission_collection.find(query))

        submissions = []
        for result in results:
            submission = {}
            submission['_id'] =  result.get('_id')
            submission['status'] = result.get('status')
            submission['user_id'] = result.get('user_id')
            submission['institute_id'] = result.get('institute_id')
            submission['created_at'] = result.get('created_at')
            submission['updated_at'] = result.get('updated_at')

            if 'clinvar_subm_id' in result:
                submission['clinvar_subm_id'] = result['clinvar_subm_id']

            if result.get('variant_data'):
                submission['variant_data'] = self.clinvar_collection.find({'_id': { "$in": result['variant_data'] } })
            if result.get('case_data'):
                submission['case_data'] = self.clinvar_collection.find({'_id' : { "$in": result['case_data'] } })

            submissions.append(submission)

        return submissions


Instruction:
How would you code a function in Python 3 to
collect a list of objects from the clinvar collection as specified by the key_id in the clinvar submission object.

Response:
def clinvar_objs(self, submission_id, key_id):
        """Collects a list of objects from the clinvar collection (variants of case data) as specified by the key_id in the clinvar submission

            Args:
                submission_id(str): the _id key of a clinvar submission
                key_id(str) : either 'variant_data' or 'case_data'. It's a key in a clinvar_submission object.
                              Its value is a list of ids of clinvar objects (either variants of casedata objects)

            Returns:
                clinvar_objects(list) : a list of clinvar objects (either variants of casedata)

        """
        # Get a submission object
        submission = self.clinvar_submission_collection.find_one({'_id': ObjectId(submission_id)})

        # a list of clinvar object ids, they can be of csv_type 'variant' or 'casedata'
        if submission.get(key_id):
            clinvar_obj_ids = list(submission.get(key_id))
            clinvar_objects = self.clinvar_collection.find({'_id' : { "$in": clinvar_obj_ids }})
            return list(clinvar_objects)
        else:
            return None


Instruction:
Create a Python 3 function for
removing a variant object from the clinvar database and update the relative submission object

Response:
def delete_clinvar_object(self, object_id, object_type, submission_id):
        """Remove a variant object from clinvar database and update the relative submission object

            Args:
                object_id(str) : the id of an object to remove from clinvar_collection database collection (a variant of a case)
                object_type(str) : either 'variant_data' or 'case_data'. It's a key in the clinvar_submission object.
                submission_id(str): the _id key of a clinvar submission

            Returns:
                updated_submission(obj): an updated clinvar submission
        """

        LOG.info("Deleting clinvar object %s (%s)", object_id, object_type)

        # If it's a variant object to be removed:
        #   remove reference to it in the submission object 'variant_data' list field
        #   remove the variant object from clinvar collection
        #   remove casedata object from clinvar collection
        #   remove reference to it in the submission object 'caset_data' list field

        # if it's a casedata object to be removed:
        #   remove reference to it in the submission object 'caset_data' list field
        #   remove casedata object from clinvar collection

        result = ''

        if object_type == 'variant_data':
            # pull out a variant from submission object
            self.clinvar_submission_collection.find_one_and_update( {'_id': ObjectId(submission_id)}, {'$pull': {'variant_data': object_id} })

            variant_object = self.clinvar_collection.find_one( {'_id': object_id} )
            linking_id = variant_object.get("linking_id") #it's the original ID of the variant in scout, it's linking clinvar variants and casedata objects together

            # remove any object with that linking_id from clinvar_collection. This removes variant and casedata
            result = self.clinvar_collection.delete_many( {'linking_id': linking_id } )

        else: # remove case_data but keep variant in submission
            # delete the object itself from clinvar_collection
             result = self.clinvar_collection.delete_one( {'_id': object_id } )

        # in any case remove reference to it in the submission object 'caset_data' list field
        self.clinvar_submission_collection.find_one_and_update( {'_id': ObjectId(submission_id)}, {'$pull': {'case_data': object_id} })

        updated_submission = self.clinvar_submission_collection.find_one_and_update( {'_id':submission_id}, { '$set' : {'updated_at': datetime.now()} }, return_document=pymongo.ReturnDocument.AFTER )

        return updated_submission


Instruction:
Can you implement a function in Python 3 that
gets all variants included in clinvar submissions for a case

Response:
def case_to_clinVars(self, case_id):
        """Get all variants included in clinvar submissions for a case

        Args:
            case_id(str): a case _id

        Returns:
            submission_variants(dict): keys are variant ids and values are variant submission objects

        """
        query = dict(case_id=case_id, csv_type='variant')
        clinvar_objs = list(self.clinvar_collection.find(query))
        submitted_vars = {}
        for clinvar in clinvar_objs:
            submitted_vars[clinvar.get('local_id')] = clinvar

        return submitted_vars


Instruction:
Create a Python 3 function to
parse a single hpo phenotype line into a dictionary.

Response:
def parse_hpo_phenotype(hpo_line):
    """Parse hpo phenotype

        Args:
            hpo_line(str): A iterable with hpo phenotype lines

        Yields:
            hpo_info(dict)
    """
    hpo_line = hpo_line.rstrip().split('\t')
    hpo_info = {}
    hpo_info['hpo_id'] = hpo_line[0]
    hpo_info['description'] = hpo_line[1]
    hpo_info['hgnc_symbol'] = hpo_line[3]

    return hpo_info


Instruction:
Given the following Python 3 function, write the documentation
def parse_hpo_gene(hpo_line):
    if not len(hpo_line) > 3:
        return {}
    hpo_line = hpo_line.rstrip().split('\t')
    hpo_info = {}
    hpo_info['hgnc_symbol'] = hpo_line[1]
    hpo_info['description'] = hpo_line[2]
    hpo_info['hpo_id'] = hpo_line[3]

    return hpo_info

Response: Parse the HPO gene information from a line of text.


Instruction:
Implement a Python 3 function for
parsing HPO disease line into dict.

Response:
def parse_hpo_disease(hpo_line):
    """Parse hpo disease line

        Args:
            hpo_line(str)
    """
    hpo_line = hpo_line.rstrip().split('\t')
    hpo_info = {}
    disease = hpo_line[0].split(':')

    hpo_info['source'] = disease[0]
    hpo_info['disease_nr'] = int(disease[1])
    hpo_info['hgnc_symbol'] = None
    hpo_info['hpo_term'] = None

    if len(hpo_line) >= 3:
        hpo_info['hgnc_symbol'] = hpo_line[2]

        if len(hpo_line) >= 4:
            hpo_info['hpo_term'] = hpo_line[3]

    return hpo_info


Instruction:
Can you generate the documentation for the following Python 3 function
def parse_hpo_phenotypes(hpo_lines):
    hpo_terms = {}
    LOG.info("Parsing hpo phenotypes...")
    for index, line in enumerate(hpo_lines):
        if index > 0 and len(line) > 0:
            hpo_info = parse_hpo_phenotype(line)
            hpo_term = hpo_info['hpo_id']
            hgnc_symbol = hpo_info['hgnc_symbol']
            if hpo_term in hpo_terms:
                hpo_terms[hpo_term]['hgnc_symbols'].append(hgnc_symbol)
            else:
                hpo_terms[hpo_term] = {
                    'hpo_id':hpo_term,
                    'description': hpo_info['description'],
                    'hgnc_symbols': [hgnc_symbol]
                }
    LOG.info("Parsing done.")
    return hpo_terms

Response: Parse the lines in the hpo phenotype file and return a dictionary with the keys and terms as keys and values as values.


Instruction:
Can you generate the documentation for the following Python 3 function
def parse_hpo_diseases(hpo_lines):
    diseases = {}
    LOG.info("Parsing hpo diseases...")
    for index, line in enumerate(hpo_lines):
        # First line is a header
        if index == 0:
            continue
        # Skip empty lines
        if not len(line) > 3:
            continue
        # Parse the info
        disease_info = parse_hpo_disease(line)
        # Skip the line if there where no info
        if not disease_info:
            continue
        disease_nr = disease_info['disease_nr']
        hgnc_symbol = disease_info['hgnc_symbol']
        hpo_term = disease_info['hpo_term']
        source = disease_info['source']
        disease_id = "{0}:{1}".format(source, disease_nr)

        if disease_id not in diseases:
            diseases[disease_id] = {
                'disease_nr': disease_nr,
                'source': source,
                'hgnc_symbols': set(),
                'hpo_terms': set(),
            }

        if hgnc_symbol:
            diseases[disease_id]['hgnc_symbols'].add(hgnc_symbol)
        if hpo_term:
            diseases[disease_id]['hpo_terms'].add(hpo_term)

    LOG.info("Parsing done.")
    return diseases

Response: Parse the HPO disease phenotypes


Instruction:
Can you generate the documentation for the following Python 3 function
def parse_hpo_to_genes(hpo_lines):
    for line in hpo_lines:
        if line.startswith('#') or len(line) < 1:
            continue
        line = line.rstrip().split('\t')
        hpo_id = line[0]
        hgnc_symbol = line[3]

        yield {
            'hpo_id': hpo_id,
            'hgnc_symbol': hgnc_symbol
        }

Response: Parse the map from a HPO term to a HGNC symbol.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def parse_hpo_genes(hpo_lines):
    LOG.info("Parsing HPO genes ...")
    genes = {}
    for index, line in enumerate(hpo_lines):
        # First line is header
        if index == 0:
            continue
        if len(line) < 5:
            continue
        gene_info = parse_hpo_gene(line)
        hgnc_symbol = gene_info['hgnc_symbol']
        description = gene_info['description']

        if hgnc_symbol not in genes:
            genes[hgnc_symbol] = {
                'hgnc_symbol': hgnc_symbol
            }

        gene = genes[hgnc_symbol]
        if description == 'Incomplete penetrance':
            gene['incomplete_penetrance'] = True
        if description == 'Autosomal dominant inheritance':
            gene['ad'] = True
        if description == 'Autosomal recessive inheritance':
            gene['ar'] = True
        if description == 'Mithochondrial inheritance':
            gene['mt'] = True
        if description == 'X-linked dominant inheritance':
            gene['xd'] = True
        if description == 'X-linked recessive inheritance':
            gene['xr'] = True
        if description == 'Y-linked inheritance':
            gene['x'] = True
        if description == 'X-linked inheritance':
            gene['y'] = True
    LOG.info("Parsing done.")
    return genes

Response: Parse HPO gene information into a dictionary with keys as keys


Instruction:
Implement a Python 3 function for
getting a set with all genes that have incomplete penetrance according to HPO

Response:
def get_incomplete_penetrance_genes(hpo_lines):
    """Get a set with all genes that have incomplete penetrance according to HPO

    Args:
        hpo_lines(iterable(str))

    Returns:
        incomplete_penetrance_genes(set): A set with the hgnc symbols of all
                                          genes with incomplete penetrance

    """
    genes = parse_hpo_genes(hpo_lines)
    incomplete_penetrance_genes = set()
    for hgnc_symbol in genes:
        if genes[hgnc_symbol].get('incomplete_penetrance'):
            incomplete_penetrance_genes.add(hgnc_symbol)
    return incomplete_penetrance_genes


Instruction:
How would you implement a function in Python 3 that
parses a. obo formated hpo line and yield a list of dicts

Response:
def parse_hpo_obo(hpo_lines):
    """Parse a .obo formated hpo line"""
    term = {}
    for line in hpo_lines:
        if len(line) == 0:
            continue
        line = line.rstrip()
        # New term starts with [Term]
        if line == '[Term]':
            if term:
                yield term
            term = {}

        elif line.startswith('id'):
            term['hpo_id'] = line[4:]

        elif line.startswith('name'):
            term['description'] = line[6:]

        elif line.startswith('alt_id'):
            if 'aliases' not in term:
                term['aliases'] = []
            term['aliases'].append(line[8:])

        elif line.startswith('is_a'):
            if 'ancestors' not in term:
                term['ancestors'] = []
            term['ancestors'].append(line[6:16])

    if term:
        yield term


Instruction:
Write a Python 3 function for
rendering seach box for genes.

Response:
def genes():
    """Render seach box for genes."""
    query = request.args.get('query', '')
    if '|' in query:
        hgnc_id = int(query.split(' | ', 1)[0])
        return redirect(url_for('.gene', hgnc_id=hgnc_id))
    gene_q = store.all_genes().limit(20)
    return dict(genes=gene_q)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def gene(hgnc_id=None, hgnc_symbol=None):
    if hgnc_symbol:
        query = store.hgnc_genes(hgnc_symbol)
        if query.count() == 1:
            hgnc_id = query.first()['hgnc_id']
        else:
            return redirect(url_for('.genes', query=hgnc_symbol))
    try:
        genes = controllers.gene(store, hgnc_id)
    except ValueError as error:
        return abort(404)

    return genes

Response: Render information about a gene.


Instruction:
How would you code a function in Python 3 to
return JSON data about genes.

Response:
def api_genes():
    """Return JSON data about genes."""
    query = request.args.get('query')
    json_out = controllers.genes_to_json(store, query)
    return jsonify(json_out)


Instruction:
Implement a Python 3 function for
checking if the panels exist in the database

Response:
def check_panels(adapter, panels, default_panels=None):
    """Make sure that the gene panels exist in the database
        Also check if the default panels are defined in gene panels

        Args:
            adapter(MongoAdapter)
            panels(list(str)): A list with panel names

        Returns:
            panels_exists(bool)
    """
    default_panels = default_panels or []
    panels_exist = True
    for panel in default_panels:
        if panel not in panels:
            log.warning("Default panels have to be defined in panels")
            panels_exist = False
    for panel in panels:
        if not adapter.gene_panel(panel):
            log.warning("Panel {} does not exist in database".format(panel))
            panels_exist = False
    return panels_exist


Instruction:
Write a Python 3 script to
load all variants in a region defined by a HGNC id

Response:
def load_region(adapter, case_id, hgnc_id=None, chrom=None, start=None, end=None):
    """Load all variants in a region defined by a HGNC id

    Args:
        adapter (MongoAdapter)
        case_id (str): Case id
        hgnc_id (int): If all variants from a gene should be uploaded
        chrom (str): If variants from coordinates should be uploaded
        start (int): Start position for region
        end (int): Stop position for region
    """
    if hgnc_id:
        gene_obj = adapter.hgnc_gene(hgnc_id)
        if not gene_obj:
            ValueError("Gene {} does not exist in database".format(hgnc_id))
        chrom = gene_obj['chromosome']
        start = gene_obj['start']
        end = gene_obj['end']

    case_obj = adapter.case(case_id=case_id)
    if not case_obj:
        raise ValueError("Case {} does not exist in database".format(case_id))

    log.info("Load clinical SNV variants for case: {0} region: chr {1}, start"
             " {2}, end {3}".format(case_obj['_id'], chrom, start, end))

    adapter.load_variants(case_obj=case_obj, variant_type='clinical',
                          category='snv', chrom=chrom, start=start, end=end)

    vcf_sv_file = case_obj['vcf_files'].get('vcf_sv')
    if vcf_sv_file:
        log.info("Load clinical SV variants for case: {0} region: chr {1}, "
                 "start {2}, end {3}".format(case_obj['_id'], chrom, start, end))
        adapter.load_variants(case_obj=case_obj, variant_type='clinical',
                              category='sv', chrom=chrom, start=start, end=end)

    vcf_str_file = case_obj['vcf_files'].get('vcf_str')
    if vcf_str_file:
        log.info("Load clinical STR variants for case: {0} region: chr {1}, "
                 "start {2}, end {3}".format(case_obj['_id'], chrom, start, end))
        adapter.load_variants(case_obj=case_obj, variant_type='clinical',
                              category='str', chrom=chrom, start=start, end=end)

    if case_obj['is_research']:
        log.info("Load research SNV variants for case: {0} region: chr {1}, "
                 "start {2}, end {3}".format(case_obj['_id'], chrom, start, end))
        adapter.load_variants(case_obj=case_obj, variant_type='research',
                              category='snv', chrom=chrom, start=start, end=end)

        vcf_sv_research = case_obj['vcf_files'].get('vcf_sv_research')
        if vcf_sv_research:
            log.info("Load research SV variants for case: {0} region: chr {1},"
                     " start {2}, end {3}".format(case_obj['_id'], chrom, start, end))
            adapter.load_variants(case_obj=case_obj, variant_type='research',
                                  category='sv', chrom=chrom, start=start, end=end)


Instruction:
Can you generate the documentation for the following Python 3 function
def load_scout(adapter, config, ped=None, update=False):
    log.info("Check that the panels exists")
    if not check_panels(adapter, config.get('gene_panels', []),
                        config.get('default_gene_panels')):
        raise ConfigError("Some panel(s) does not exist in the database")
    case_obj = adapter.load_case(config, update=update)
    return case_obj

Response: Load a new case from a Scout config.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def templated(template=None):
    def decorator(f):
        @wraps(f)
        def decorated_function(*args, **kwargs):
            template_name = template
            if template_name is None:
                template_name = request.endpoint.replace('.', '/') + '.html'
            context = f(*args, **kwargs)
            if context is None:
                context = {}
            elif not isinstance(context, dict):
                return context
            return render_template(template_name, **context)
        return decorated_function
    return decorator

Response: Decorator that returns a function that renders the template with the specified name.


Instruction:
Explain what the following Python 3 code does
def institute_and_case(store, institute_id, case_name=None):
    institute_obj = store.institute(institute_id)
    if institute_obj is None and institute_id != 'favicon.ico':
        flash("Can't find institute: {}".format(institute_id), 'warning')
        return abort(404)

    if case_name:
        if case_name:
            case_obj = store.case(institute_id=institute_id, display_name=case_name)
            if case_obj is None:
                return abort(404)

    # validate that user has access to the institute
    if not current_user.is_admin:
        if institute_id not in current_user.institutes:
            if not case_name or not any(inst_id in case_obj['collaborators'] for inst_id in
                                        current_user.institutes):
                # you don't have access!!
                flash("You don't have acccess to: {}".format(institute_id),'danger')
                return abort(403)

    # you have access!
    if case_name:
        return institute_obj, case_obj
    else:
        return institute_obj

Response: Fetch institute and case objects.


Instruction:
How would you implement a function in Python 3 that
gets the hgnc id for a gene

Response:
def get_hgnc_id(gene_info, adapter):
    """Get the hgnc id for a gene

        The proprity order will be
        1. if there is a hgnc id this one will be choosen
        2. if the hgnc symbol matches a genes proper hgnc symbol
        3. if the symbol ony matches aliases on several genes one will be
           choosen at random

        Args:
            gene_info(dict)
            adapter

        Returns:
            true_id(int)
    """
    hgnc_id = gene_info.get('hgnc_id')
    hgnc_symbol = gene_info.get('hgnc_symbol')

    true_id = None

    if hgnc_id:
        true_id = int(hgnc_id)
    else:
        gene_result = adapter.hgnc_genes(hgnc_symbol)
        if gene_result.count() == 0:
            raise Exception("No gene could be found for {}".format(hgnc_symbol))
        for gene in gene_result:
            if hgnc_symbol.upper() == gene.hgnc_symbol.upper():
                true_id = gene.hgnc_id
        if not gene_info['hgnc_id']:
            true_id = gene.hgnc_id

    return true_id


Instruction:
How would you implement a function in Python 3 that
updates a panel in the database

Response:
def panel(context, panel, version, update_date, update_version):
    """
    Update a panel in the database
    """
    adapter = context.obj['adapter']

    # Check that the panel exists
    panel_obj = adapter.gene_panel(panel, version=version)

    if not panel_obj:
        LOG.warning("Panel %s (version %s) could not be found" % (panel, version))
        context.abort()

    date_obj = None
    if update_date:
        try:
            date_obj = get_date(update_date)
        except Exception as err:
            LOG.warning(err)
            context.abort()

    update_panel(
        adapter,
        panel,
        panel_version=panel_obj['version'],
        new_version=update_version,
        new_date=date_obj
    )


Instruction:
Can you create a Python 3 function that
updates the disease terms in mongo database.

Response:
def diseases(context, api_key):
    """
    Update disease terms in mongo database.
    """
    adapter = context.obj['adapter']

    # Fetch the omim information
    api_key = api_key or context.obj.get('omim_api_key')
    if not api_key:
        LOG.warning("Please provide a omim api key to load the omim gene panel")
        context.abort()

    try:
        mim_files = fetch_mim_files(api_key, genemap2=True)
    except Exception as err:
        LOG.warning(err)
        context.abort()

    LOG.info("Dropping DiseaseTerms")
    adapter.disease_term_collection.drop()
    LOG.debug("DiseaseTerms dropped")

    load_disease_terms(
        adapter=adapter,
        genemap_lines=mim_files['genemap2'],
    )

    LOG.info("Successfully loaded all disease terms")


Instruction:
How would you code a function in Python 3 to
load the hpo terms and diseases into the database.

Response:
def load_hpo(adapter, disease_lines, hpo_disease_lines=None, hpo_lines=None, hpo_gene_lines=None):
    """Load the hpo terms and hpo diseases into database

    Args:
        adapter(MongoAdapter)
        disease_lines(iterable(str)): These are the omim genemap2 information
        hpo_lines(iterable(str))
        disease_lines(iterable(str))
        hpo_gene_lines(iterable(str))
    """
    # Create a map from gene aliases to gene objects
    alias_genes = adapter.genes_by_alias()

    # Fetch the hpo terms if no file
    if not hpo_lines:
        hpo_lines = fetch_hpo_terms()

    # Fetch the hpo gene information if no file
    if not hpo_gene_lines:
        hpo_gene_lines = fetch_hpo_to_genes()

    # Fetch the hpo phenotype information if no file
    if not hpo_disease_lines:
        hpo_disease_lines = fetch_hpo_phenotype_to_terms()

    load_hpo_terms(adapter, hpo_lines, hpo_gene_lines, alias_genes)

    load_disease_terms(adapter, disease_lines, alias_genes, hpo_disease_lines)


Instruction:
Create a Python 3 function to
load the hpo terms into the database and add them to the database

Response:
def load_hpo_terms(adapter, hpo_lines=None, hpo_gene_lines=None, alias_genes=None):
    """Load the hpo terms into the database

    Parse the hpo lines, build the objects and add them to the database

    Args:
        adapter(MongoAdapter)
        hpo_lines(iterable(str))
        hpo_gene_lines(iterable(str))
    """

    # Store the hpo terms
    hpo_terms = {}

    # Fetch the hpo terms if no file
    if not hpo_lines:
        hpo_lines = fetch_hpo_terms()

    # Fetch the hpo gene information if no file
    if not hpo_gene_lines:
        hpo_gene_lines = fetch_hpo_to_genes()

    # Parse the terms
    # This will yield dictionaries with information about the terms
    LOG.info("Parsing hpo terms")
    for term in parse_hpo_obo(hpo_lines):
        hpo_terms[term['hpo_id']] = term

    # Get a map with hgnc symbols to hgnc ids from scout
    if not alias_genes:
        alias_genes = adapter.genes_by_alias()

    LOG.info("Adding gene information to hpo terms ...")
    for hpo_to_symbol in parse_hpo_to_genes(hpo_gene_lines):
        hgnc_symbol = hpo_to_symbol['hgnc_symbol']
        hpo_id = hpo_to_symbol['hpo_id']

        # Fetch gene info to get correct hgnc id
        gene_info = alias_genes.get(hgnc_symbol)
        if not gene_info:
            continue

        hgnc_id = gene_info['true']

        if hpo_id not in hpo_terms:
            continue

        hpo_term = hpo_terms[hpo_id]

        if not 'genes' in hpo_term:
            hpo_term['genes'] = set()

        hpo_term['genes'].add(hgnc_id)

    start_time = datetime.now()

    LOG.info("Loading the hpo terms...")
    nr_terms = len(hpo_terms)
    hpo_bulk = []
    with progressbar(hpo_terms.values(), label="Loading hpo terms", length=nr_terms) as bar:

        for hpo_info in bar:
            hpo_bulk.append(build_hpo_term(hpo_info))

        if len(hpo_bulk) > 10000:
            adapter.load_hpo_bulk(hpo_bulk)
            hpo_bulk = []

    if hpo_bulk:
        adapter.load_hpo_bulk(hpo_bulk)

    LOG.info("Loading done. Nr of terms loaded {0}".format(nr_terms))
    LOG.info("Time to load terms: {0}".format(datetime.now() - start_time))


Instruction:
Implement a Python 3 function for
loading the omim phenotypes into the database

Response:
def load_disease_terms(adapter, genemap_lines, genes=None, hpo_disease_lines=None):
    """Load the omim phenotypes into the database

    Parse the phenotypes from genemap2.txt and find the associated hpo terms
    from ALL_SOURCES_ALL_FREQUENCIES_diseases_to_genes_to_phenotypes.txt.

    Args:
        adapter(MongoAdapter)
        genemap_lines(iterable(str))
        genes(dict): Dictionary with all genes found in database
        hpo_disease_lines(iterable(str))

    """
    # Get a map with hgnc symbols to hgnc ids from scout
    if not genes:
        genes = adapter.genes_by_alias()

    # Fetch the disease terms from omim
    disease_terms = get_mim_phenotypes(genemap_lines=genemap_lines)

    if not hpo_disease_lines:
        hpo_disease_lines = fetch_hpo_phenotype_to_terms()
    hpo_diseases = parse_hpo_diseases(hpo_disease_lines)

    start_time = datetime.now()
    nr_diseases = None

    LOG.info("Loading the hpo disease...")
    for nr_diseases, disease_number in enumerate(disease_terms):
        disease_info = disease_terms[disease_number]
        disease_id = "OMIM:{0}".format(disease_number)

        if disease_id in hpo_diseases:
            hpo_terms = hpo_diseases[disease_id]['hpo_terms']
            if hpo_terms:
                disease_info['hpo_terms'] = hpo_terms
        disease_obj = build_disease_term(disease_info, genes)

        adapter.load_disease_term(disease_obj)

    LOG.info("Loading done. Nr of diseases loaded {0}".format(nr_diseases))
    LOG.info("Time to load diseases: {0}".format(datetime.now() - start_time))


Instruction:
Can you tell what is the following Python 3 function doing
def parse_frequencies(variant, transcripts):
    frequencies = {}
    # These lists could be extended...
    thousand_genomes_keys = ['1000GAF']
    thousand_genomes_max_keys = ['1000G_MAX_AF']

    exac_keys = ['EXACAF']
    exac_max_keys = ['ExAC_MAX_AF', 'EXAC_MAX_AF']

    gnomad_keys = ['GNOMADAF', 'GNOMAD_AF']
    gnomad_max_keys = ['GNOMADAF_POPMAX', 'GNOMADAF_MAX']

    for test_key in thousand_genomes_keys:
        thousand_g = parse_frequency(variant, test_key)
        if thousand_g:
            frequencies['thousand_g'] = thousand_g
            break

    for test_key in thousand_genomes_max_keys:
        thousand_g_max = parse_frequency(variant, test_key)
        if thousand_g_max:
            frequencies['thousand_g_max'] = thousand_g_max
            break

    for test_key in exac_keys:
        exac = parse_frequency(variant, test_key)
        if exac:
            frequencies['exac'] = exac
            break

    for test_key in exac_max_keys:
        exac_max = parse_frequency(variant, test_key)
        if exac_max:
            frequencies['exac_max'] = exac_max
            break

    for test_key in gnomad_keys:
        gnomad = parse_frequency(variant, test_key)
        if gnomad:
            frequencies['gnomad'] = gnomad
            break

    for test_key in gnomad_max_keys:
        gnomad_max = parse_frequency(variant, test_key)
        if gnomad_max:
            frequencies['gnomad_max'] = gnomad_max
            break

    # Search transcripts if not found in VCF
    if not frequencies:
        for transcript in transcripts:
            exac = transcript.get('exac_maf')
            exac_max = transcript.get('exac_max')

            thousand_g = transcript.get('thousand_g_maf')
            thousandg_max = transcript.get('thousandg_max')

            gnomad = transcript.get('gnomad_maf')
            gnomad_max = transcript.get('gnomad_max')
            if exac:
                frequencies['exac'] = exac
            if exac_max:
                frequencies['exac_max'] = exac_max
            if thousand_g:
                frequencies['thousand_g'] = thousand_g
            if thousandg_max:
                frequencies['thousand_g_max'] = thousandg_max
            if gnomad:
                frequencies['gnomad'] = gnomad
            if gnomad_max:
                frequencies['gnomad_max'] = gnomad_max

    #These are SV-specific frequencies
    thousand_g_left = parse_frequency(variant, 'left_1000GAF')
    if thousand_g_left:
        frequencies['thousand_g_left'] = thousand_g_left

    thousand_g_right = parse_frequency(variant, 'right_1000GAF')
    if thousand_g_right:
        frequencies['thousand_g_right'] = thousand_g_right

    return frequencies

Response:
Add the frequencies to the variant
Taxonomy


Instruction:
Create a Python 3 function to
parse any frequency from the info dict

Response:
def parse_frequency(variant, info_key):
    """Parse any frequency from the info dict

    Args:
        variant(cyvcf2.Variant)
        info_key(str)

    Returns:
        frequency(float): or None if frequency does not exist
    """
    raw_annotation = variant.INFO.get(info_key)
    raw_annotation = None if raw_annotation == '.' else raw_annotation
    frequency = float(raw_annotation) if raw_annotation else None
    return frequency


Instruction:
How would you code a function in Python 3 to
parse the custom sv frequencies for a given structural variant.

Response:
def parse_sv_frequencies(variant):
    """Parsing of some custom sv frequencies

    These are very specific at the moment, this will hopefully get better over time when the
    field of structural variants is more developed.

    Args:
        variant(cyvcf2.Variant)

    Returns:
        sv_frequencies(dict)
    """
    frequency_keys = [
        'clingen_cgh_benignAF',
        'clingen_cgh_benign',
        'clingen_cgh_pathogenicAF',
        'clingen_cgh_pathogenic',
        'clingen_ngi',
        'clingen_ngiAF',
        'swegen',
        'swegenAF',
        'decipherAF',
        'decipher'
    ]
    sv_frequencies = {}

    for key in frequency_keys:
        value = variant.INFO.get(key, 0)
        if 'AF' in key:
            value = float(value)
        else:
            value = int(value)
        if value > 0:
            sv_frequencies[key] = value
    return sv_frequencies


Instruction:
Write a Python 3 function that can
show all users in the database

Response:
def users(context):
    """Show all users in the database"""
    LOG.info("Running scout view users")
    adapter = context.obj['adapter']

    user_objs = adapter.users()
    if user_objs.count() == 0:
        LOG.info("No users found")
        context.abort()

    click.echo("#name\temail\troles\tinstitutes")
    for user_obj in user_objs:
        click.echo("{0}\t{1}\t{2}\t{3}\t".format(
            user_obj['name'],
            user_obj.get('mail', user_obj['_id']),
            ', '.join(user_obj.get('roles', [])),
            ', '.join(user_obj.get('institutes', [])),
        )
        )


Instruction:
Given the following Python 3 function, write the documentation
def load_case(adapter, case_obj, update=False):
    logger.info('Loading case {} into database'.format(case_obj['display_name']))

    # Check if case exists in database
    existing_case = adapter.case(case_obj['_id'])

    if existing_case:
        if update:
            adapter.update_case(case_obj)
        else:
            raise IntegrityError("Case {0} already exists in database".format(case_obj['_id']))
    else:
        adapter.add_case(case_obj)
    return case_obj

Response: Load a case into the database


Instruction:
Write a Python 3 function for
building a HGNC gene object from a dictionary of gene information.

Response:
def build_hgnc_gene(gene_info, build='37'):
    """Build a hgnc_gene object

        Args:
            gene_info(dict): Gene information

        Returns:
            gene_obj(dict)

            {
                '_id': ObjectId(),
                # This is the hgnc id, required:
                'hgnc_id': int,
                # The primary symbol, required
                'hgnc_symbol': str,
                'ensembl_id': str, # required
                'build': str, # '37' or '38', defaults to '37', required

                'chromosome': str, # required
                'start': int, # required
                'end': int, # required

                'description': str, # Gene description
                'aliases': list(), # Gene symbol aliases, includes hgnc_symbol, str
                'entrez_id': int,
                'omim_id': int,
                'pli_score': float,
                'primary_transcripts': list(), # List of refseq transcripts (str)
                'ucsc_id': str,
                'uniprot_ids': list(), # List of str
                'vega_id': str,
                'transcripts': list(), # List of hgnc_transcript

                # Inheritance information
                'inheritance_models': list(), # List of model names
                'incomplete_penetrance': bool, # Acquired from HPO

                # Phenotype information
                'phenotypes': list(), # List of dictionaries with phenotype information
            }
    """
    try:
        hgnc_id = int(gene_info['hgnc_id'])
    except KeyError as err:
        raise KeyError("Gene has to have a hgnc_id")
    except ValueError as err:
        raise ValueError("hgnc_id has to be integer")

    try:
        hgnc_symbol = gene_info['hgnc_symbol']
    except KeyError as err:
        raise KeyError("Gene has to have a hgnc_symbol")

    try:
        ensembl_id = gene_info['ensembl_gene_id']
    except KeyError as err:
        raise KeyError("Gene has to have a ensembl_id")

    try:
        chromosome = gene_info['chromosome']
    except KeyError as err:
        raise KeyError("Gene has to have a chromosome")

    try:
        start = int(gene_info['start'])
    except KeyError as err:
        raise KeyError("Gene has to have a start position")
    except TypeError as err:
        raise TypeError("Gene start has to be a integer")

    try:
        end = int(gene_info['end'])
    except KeyError as err:
        raise KeyError("Gene has to have a end position")
    except TypeError as err:
        raise TypeError("Gene end has to be a integer")

    gene_obj = HgncGene(
        hgnc_id=hgnc_id,
        hgnc_symbol=hgnc_symbol,
        ensembl_id=ensembl_id,
        chrom=chromosome,
        start=start,
        end=end,
        build=build,
    )

    if gene_info.get('description'):
        gene_obj['description'] = gene_info['description']
        # LOG.debug("Adding info %s", gene_info['description'])

    if gene_info.get('previous_symbols'):
        gene_obj['aliases'] = gene_info['previous_symbols']

    if gene_info.get('entrez_id'):
        gene_obj['entrez_id'] = int(gene_info['entrez_id'])

    if gene_info.get('omim_id'):
        gene_obj['omim_id'] = int(gene_info['omim_id'])

    if gene_info.get('pli_score'):
        gene_obj['pli_score'] = float(gene_info['pli_score'])

    if gene_info.get('ref_seq'):
        gene_obj['primary_transcripts'] = gene_info['ref_seq']

    if gene_info.get('ucsc_id'):
        gene_obj['ucsc_id'] = gene_info['ucsc_id']

    if gene_info.get('uniprot_ids'):
        gene_obj['uniprot_ids'] = gene_info['uniprot_ids']

    if gene_info.get('vega_id'):
        gene_obj['vega_id'] = gene_info['vega_id']

    if gene_info.get('incomplete_penetrance'):
        gene_obj['incomplete_penetrance'] = True

    if gene_info.get('inheritance_models'):
        gene_obj['inheritance_models'] = gene_info['inheritance_models']

    phenotype_objs = []
    for phenotype_info in gene_info.get('phenotypes', []):
        phenotype_objs.append(build_phenotype(phenotype_info))

    if phenotype_objs:
        gene_obj['phenotypes'] = phenotype_objs

    for key in list(gene_obj):
        if gene_obj[key] is None:
            gene_obj.pop(key)

    return gene_obj


Instruction:
Write a Python 3 script for
loading a gene panel based on the info sent

Response:
def load_panel(self, parsed_panel):
        """Load a gene panel based on the info sent
        A panel object is built and integrity checks are made.
        The panel object is then loaded into the database.

        Args:
            path(str): Path to panel file
            institute(str): Name of institute that owns the panel
            panel_id(str): Panel id
            date(datetime.datetime): Date of creation
            version(float)
            full_name(str): Option to have a long name

            panel_info(dict): {
                'file': <path to panel file>(str),
                'institute': <institute>(str),
                'type': <panel type>(str),
                'date': date,
                'version': version,
                'panel_name': panel_id,
                'full_name': name,
            }
        """
        panel_obj = build_panel(parsed_panel, self)

        self.add_gene_panel(panel_obj)


Instruction:
Create a Python 3 function to
create and load the OMIM - AUTO panel

Response:
def load_omim_panel(self, api_key, institute=None):
        """Create and load the OMIM-AUTO panel"""
        existing_panel = self.gene_panel(panel_id='OMIM-AUTO')
        if not existing_panel:
            LOG.warning("OMIM-AUTO does not exists in database")
            LOG.info('Creating a first version')
            version = 1.0

        if existing_panel:
            version = float(math.floor(existing_panel['version']) + 1)

        LOG.info("Setting version to %s", version)

        try:
            mim_files = fetch_mim_files(api_key=api_key, genemap2=True, mim2genes=True)
        except Exception as err:
            raise err

        date_string = None
        # Get the correct date when omim files where released
        for line in mim_files['genemap2']:
            if 'Generated' in line:
                date_string = line.split(':')[-1].lstrip().rstrip()
        date_obj = get_date(date_string)

        if existing_panel:
            if existing_panel['date'] == date_obj:
                LOG.warning("There is no new version of OMIM")
                return

        panel_data = {}
        panel_data['path'] = None
        panel_data['type'] = 'clinical'
        panel_data['date'] = date_obj
        panel_data['panel_id'] = 'OMIM-AUTO'
        panel_data['institute'] = institute or 'cust002'
        panel_data['version'] = version
        panel_data['display_name'] = 'OMIM-AUTO'
        panel_data['genes'] = []

        alias_genes = self.genes_by_alias()

        genes = get_omim_panel_genes(
            genemap2_lines = mim_files['genemap2'],
            mim2gene_lines = mim_files['mim2genes'],
            alias_genes = alias_genes,
        )

        for gene in genes:
            panel_data['genes'].append(gene)

        panel_obj = build_panel(panel_data, self)

        if existing_panel:

            new_genes = self.compare_mim_panels(existing_panel, panel_obj)
            if new_genes:
                self.update_mim_version(new_genes, panel_obj, old_version=existing_panel['version'])
            else:
                LOG.info("The new version of omim does not differ from the old one")
                LOG.info("No update is added")
                return

        self.add_gene_panel(panel_obj)


Instruction:
Can you write a function in Python 3 where it
checks if the latest version of OMIM differs from the most recent in database Return all genes that where not in the previous version

Response:
def compare_mim_panels(self, existing_panel, new_panel):
        """Check if the latest version of OMIM differs from the most recent in database
           Return all genes that where not in the previous version.

        Args:
            existing_panel(dict)
            new_panel(dict)

        Returns:
            new_genes(set(str))
        """
        existing_genes = set([gene['hgnc_id'] for gene in existing_panel['genes']])
        new_genes = set([gene['hgnc_id'] for gene in new_panel['genes']])

        return new_genes.difference(existing_genes)


Instruction:
Implement a Python 3 function for
updating the correct version for each gene in the new panel with the new version.

Response:
def update_mim_version(self, new_genes, new_panel, old_version):
        """Set the correct version for each gene
        Loop over the genes in the new panel

        Args:
            new_genes(set(str)): Set with the new gene symbols
            new_panel(dict)

        """
        LOG.info('Updating versions for new genes')
        version = new_panel['version']
        for gene in new_panel['genes']:
            gene_symbol = gene['hgnc_id']
            # If the gene is new we add the version
            if gene_symbol in new_genes:
                gene['database_entry_version'] = version
                continue
            # If the gene is old it will have the previous version
            gene['database_entry_version'] = old_version

        return


Instruction:
Make a summary of the following Python 3 code
def add_gene_panel(self, panel_obj):
        panel_name = panel_obj['panel_name']
        panel_version = panel_obj['version']
        display_name = panel_obj.get('display_name', panel_name)

        if self.gene_panel(panel_name, panel_version):
            raise IntegrityError("Panel {0} with version {1} already"
                                 " exist in database".format(panel_name, panel_version))
        LOG.info("loading panel {0}, version {1} to database".format(
            display_name, panel_version
        ))
        result = self.panel_collection.insert_one(panel_obj)
        LOG.debug("Panel saved")
        return result.inserted_id

Response: Add a gene panel to the database. Returns the ID of the newly added gene panel.


Instruction:
Can you generate the documentation for the following Python 3 function
def panel(self, panel_id):
        if not isinstance(panel_id, ObjectId):
            panel_id = ObjectId(panel_id)
        panel_obj = self.panel_collection.find_one({'_id': panel_id})
        return panel_obj

Response: Fetch a gene panel by _id.


Instruction:
Write a Python 3 script to
delete a panel by _id.

Response:
def delete_panel(self, panel_obj):
        """Delete a panel by '_id'.

        Args:
            panel_obj(dict)

        Returns:
            res(pymongo.DeleteResult)
        """
        res = self.panel_collection.delete_one({'_id': panel_obj['_id']})
        LOG.warning("Deleting panel %s, version %s" % (panel_obj['panel_name'], panel_obj['version']))
        return res


Instruction:
Write a Python 3 function that can
fetch a gene panel object from the database.

Response:
def gene_panel(self, panel_id, version=None):
        """Fetch a gene panel.

        If no panel is sent return all panels

        Args:
            panel_id (str): unique id for the panel
            version (str): version of the panel. If 'None' latest version will be returned

        Returns:
            gene_panel: gene panel object
        """
        query = {'panel_name': panel_id}
        if version:
            LOG.info("Fetch gene panel {0}, version {1} from database".format(
                panel_id, version
            ))
            query['version'] = version
            return self.panel_collection.find_one(query)
        else:
            LOG.info("Fetching gene panels %s from database", panel_id)
            res = self.panel_collection.find(query).sort('version', -1)
            if res.count() > 0:
                return res[0]
            else:
                LOG.info("No gene panel found")
                return None


Instruction:
How would you implement a function in Python 3 that
returns all gene panels by that panel name. If panel_id return all versions of panels by that panel name. If institute_id return all versions of panels by that panel name. If version return all versions of panels by that panel name. If institute_id return all versions of panels by that panel name.

Response:
def gene_panels(self, panel_id=None, institute_id=None, version=None):
        """Return all gene panels

        If panel_id return all versions of panels by that panel name

        Args:
            panel_id(str)

        Returns:
            cursor(pymongo.cursor)
        """
        query = {}
        if panel_id:
            query['panel_name'] = panel_id
            if version:
                query['version'] = version
        if institute_id:
            query['institute'] = institute_id

        return self.panel_collection.find(query)


Instruction:
How would you explain what the following Python 3 function does
def gene_to_panels(self, case_obj):
        LOG.info("Building gene to panels")
        gene_dict = {}

        for panel_info in case_obj.get('panels', []):
            panel_name = panel_info['panel_name']
            panel_version = panel_info['version']
            panel_obj = self.gene_panel(panel_name, version=panel_version)
            if not panel_obj:
                ## Raise exception here???
                LOG.warning("Panel: {0}, version {1} does not exist in database".format(panel_name, panel_version))

            for gene in panel_obj['genes']:
                hgnc_id = gene['hgnc_id']

                if hgnc_id not in gene_dict:
                    gene_dict[hgnc_id] = set([panel_name])
                    continue

                gene_dict[hgnc_id].add(panel_name)

        LOG.info("Gene to panels done")

        return gene_dict

Response: Fetch all gene panels and group them by gene


Instruction:
Make a summary of the following Python 3 code
def update_panel(self, panel_obj, version=None, date_obj=None):
        LOG.info("Updating panel %s", panel_obj['panel_name'])
        # update date of panel to "today"
        date = panel_obj['date']
        if version:
            LOG.info("Updating version from {0} to version {1}".format(
                panel_obj['version'], version))
            panel_obj['version'] = version
            # Updating version should not update date
            if date_obj:
                date = date_obj
        else:
            date = date_obj or dt.datetime.now()
        panel_obj['date'] = date

        updated_panel = self.panel_collection.find_one_and_replace(
            {'_id': panel_obj['_id']},
            panel_obj,
            return_document=pymongo.ReturnDocument.AFTER
        )

        return updated_panel

Response: Update a panel with a new one.


Instruction:
Can you write a function in Python 3 where it
adds a pending action to a panel

Response:
def add_pending(self, panel_obj, hgnc_gene, action, info=None):
        """Add a pending action to a gene panel

        Store the pending actions in panel.pending

        Args:
            panel_obj(dict): The panel that is about to be updated
            hgnc_gene(dict)
            action(str): choices=['add','delete','edit']
            info(dict): additional gene info (disease_associated_transcripts,
                        reduced_penetrance, mosaicism, database_entry_version ,
                        inheritance_models, comment)

        Returns:
            updated_panel(dict):

        """

        valid_actions = ['add', 'delete', 'edit']
        if action not in valid_actions:
            raise ValueError("Invalid action {0}".format(action))

        info = info or {}
        pending_action = {
            'hgnc_id': hgnc_gene['hgnc_id'],
            'action': action,
            'info': info,
            'symbol': hgnc_gene['hgnc_symbol'],
        }

        updated_panel = self.panel_collection.find_one_and_update(
            {'_id': panel_obj['_id']},
            {
                '$addToSet': {
                    'pending': pending_action
                }
            },
            return_document=pymongo.ReturnDocument.AFTER
        )

        return updated_panel


Instruction:
Write a Python 3 function for
applying the pending changes to an existing gene panel or create a new version of the same panel.

Response:
def apply_pending(self, panel_obj, version):
        """Apply the pending changes to an existing gene panel or create a new version of the same panel.

        Args:
            panel_obj(dict): panel in database to update
            version(double): panel version to update

        Returns:
            inserted_id(str): id of updated panel or the new one
        """

        updates = {}
        new_panel = deepcopy(panel_obj)
        new_panel['pending'] = []
        new_panel['date'] = dt.datetime.now()
        info_fields = ['disease_associated_transcripts', 'inheritance_models', 'reduced_penetrance',
            'mosaicism', 'database_entry_version', 'comment']
        new_genes = []

        for update in panel_obj.get('pending', []):
            hgnc_id = update['hgnc_id']

            # If action is add we create a new gene object
            if update['action'] != 'add':
                updates[hgnc_id] = update
                continue
            info = update.get('info', {})
            gene_obj = {
                'hgnc_id': hgnc_id,
                'symbol': update['symbol']
            }

            for field in info_fields:
                if field in info:
                    gene_obj[field] = info[field]
            new_genes.append(gene_obj)

        for gene in panel_obj['genes']:
            hgnc_id = gene['hgnc_id']

            if hgnc_id not in updates:
                new_genes.append(gene)
                continue

            current_update = updates[hgnc_id]
            action = current_update['action']
            info = current_update['info']

            # If action is delete we do not add the gene to new genes
            if action == 'delete':
                continue

            elif action == 'edit':
                for field in info_fields:
                    if field in info:
                        gene[field] = info[field]
                new_genes.append(gene)

        new_panel['genes'] = new_genes
        new_panel['version'] = float(version)

        inserted_id = None
        # if the same version of the panel should be updated
        if new_panel['version'] == panel_obj['version']:
            # replace panel_obj with new_panel
            result = self.panel_collection.find_one_and_replace(
                {'_id':panel_obj['_id']},
                new_panel,
                return_document=pymongo.ReturnDocument.AFTER
            )
            inserted_id = result['_id']
        else: # create a new version of the same panel
            new_panel.pop('_id')

            # archive the old panel
            panel_obj['is_archived'] = True
            self.update_panel(panel_obj=panel_obj, date_obj=panel_obj['date'])

            # insert the new panel
            inserted_id = self.panel_collection.insert_one(new_panel).inserted_id

        return inserted_id


Instruction:
Can you tell what is the following Python 3 function doing
def clinical_symbols(self, case_obj):
        panel_ids = [panel['panel_id'] for panel in case_obj['panels']]
        query = self.panel_collection.aggregate([
            {'$match': {'_id': {'$in': panel_ids}}},
            {'$unwind': '$genes'},
            {'$group': {'_id': '$genes.symbol'}}
        ])
        return set(item['_id'] for item in query)

Response: Return all the clinical gene symbols for a case.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def cases(context, case_id, institute, reruns, finished, causatives, research_requested,
          is_research, status, json):
    """Interact with cases existing in the database."""
    adapter = context.obj['adapter']

    models = []
    if case_id:
        case_obj = adapter.case(case_id=case_id)
        if case_obj:
            models.append(case_obj)
        else:
            LOG.info("No case with id {}".format(case_id))

    else:
        models = adapter.cases(collaborator=institute, reruns=reruns,
                           finished=finished, has_causatives=causatives,
                           research_requested=research_requested,
                           is_research=is_research, status=status)
        models = [case_obj for case_obj in models]
        if len(models) == 0:
            LOG.info("No cases could be found")

    if json:
        click.echo(dumps(models))
        return

    for model in models:
        pp(model)

Response: Interact with cases existing in the database.


Instruction:
Explain what the following Python 3 code does
def emit(self, record):
        try:
            import smtplib
            try:
                from email.utils import formatdate
            except ImportError:
                formatdate = self.date_time

            port = self.mailport
            if not port:
                port = smtplib.SMTP_PORT

            smtp = smtplib.SMTP(self.mailhost, port)
            msg = self.format(record)
            msg = "From: %s\r\nTo: %s\r\nSubject: %s\r\nDate: %s\r\n\r\n%s" % (
                self.fromaddr,
                ','.join(self.toaddrs),
                self.getSubject(record),
                formatdate(), msg
            )

            if self.username:
                smtp.ehlo()  # For 'tls', add this line
                smtp.starttls()  # For 'tls', add this line
                smtp.ehlo()  # For 'tls', add this line
                smtp.login(self.username, self.password)

            smtp.sendmail(self.fromaddr, self.toaddrs, msg)
            smtp.quit()

        except (KeyboardInterrupt, SystemExit):
            raise

        except:
            self.handleError(record)

Response: Emit a record and send it to the specified addressees.


Instruction:
How would you implement a function in Python 3 that
returns a list with the current indexes

Response:
def indexes(self, collection=None):
        """Return a list with the current indexes

        Skip the mandatory _id_ indexes

        Args:
            collection(str)

        Returns:
            indexes(list)
        """

        indexes = []

        for collection_name in self.collections():
            if collection and collection != collection_name:
                continue
            for index_name in self.db[collection_name].index_information():
                if index_name != '_id_':
                    indexes.append(index_name)
        return indexes


Instruction:
Can you write a function in Python 3 where it
loads the proper indexes for the current scout instance.

Response:
def load_indexes(self):
        """Add the proper indexes to the scout instance.

        All indexes are specified in scout/constants/indexes.py

        If this method is utilised when new indexes are defined those should be added

        """
        for collection_name in INDEXES:
            existing_indexes = self.indexes(collection_name)
            indexes = INDEXES[collection_name]
            for index in indexes:
                index_name = index.document.get('name')
                if index_name in existing_indexes:
                    LOG.info("Deleting old index: %s" % index_name)
                    self.db[collection_name].drop_index(index_name)
            LOG.info("creating indexes for {0} collection: {1}".format(
                collection_name,
                ', '.join([index.document.get('name') for index in indexes])
                ))
            self.db[collection_name].create_indexes(indexes)


Instruction:
Create a Python 3 function to
update the indexes in the database.

Response:
def update_indexes(self):
        """Update the indexes

        If there are any indexes that are not added to the database, add those.

        """
        LOG.info("Updating indexes...")
        nr_updated = 0
        for collection_name in INDEXES:
            existing_indexes = self.indexes(collection_name)
            indexes = INDEXES[collection_name]
            for index in indexes:
                index_name = index.document.get('name')
                if index_name not in existing_indexes:
                    nr_updated += 1
                    LOG.info("Adding index : %s" % index_name)
                    self.db[collection_name].create_indexes(indexes)
        if nr_updated == 0:
            LOG.info("All indexes in place")


Instruction:
Explain what the following Python 3 code does
def drop_indexes(self):
        LOG.warning("Dropping all indexe")
        for collection_name in INDEXES:
            LOG.warning("Dropping all indexes for collection name %s", collection_name)
            self.db[collection_name].drop_indexes()

Response: Delete all indexes for the database


Instruction:
Can you generate the documentation for the following Python 3 function
def build_variant_query(self, query=None, category='snv', variant_type=['clinical']):

        query = query or {}
        mongo_variant_query = {}

        LOG.debug("Building a mongo query for %s" % query)

        if query.get('hgnc_symbols'):
            mongo_variant_query['hgnc_symbols'] = {'$in': query['hgnc_symbols']}

        mongo_variant_query['variant_type'] = {'$in': variant_type}

        mongo_variant_query['category'] = category

        rank_score = query.get('rank_score') or 15

        mongo_variant_query['rank_score'] = {'$gte': rank_score}
        LOG.debug("Querying %s" % mongo_variant_query)

        return mongo_variant_query

Response: Build a mongo query across multiple cases.