Instruction:
Create a Python 3 function for
parsing the file called mim2gene returning a list of dictionaries each containing the entry type and gene ID.

Response:
def parse_mim2gene(lines):
    """Parse the file called mim2gene

    This file describes what type(s) the different mim numbers have.
    The different entry types are: 'gene', 'gene/phenotype', 'moved/removed',
    'phenotype', 'predominantly phenotypes'
    Where:
        gene: Is a gene entry
        gene/phenotype: This entry describes both a phenotype and a gene
        moved/removed: No explanation needed
        phenotype: Describes a phenotype
        predominantly phenotype: Not clearly established (probably phenotype)

    Args:
        lines(iterable(str)): The mim2gene lines

    Yields:
        parsed_entry(dict)

        {
            "mim_number": int,
            "entry_type": str,
            "entrez_gene_id": int,
            "hgnc_symbol": str,
            "ensembl_gene_id": str,
            "ensembl_transcript_id": str,
        }

    """
    LOG.info("Parsing mim2gene")
    header = ["mim_number", "entry_type", "entrez_gene_id", "hgnc_symbol", "ensembl_gene_id"]
    for i, line in enumerate(lines):
        if line.startswith('#'):
            continue

        if not len(line) > 0:
            continue

        line = line.rstrip()
        parsed_entry = parse_omim_line(line, header)
        parsed_entry['mim_number'] = int(parsed_entry['mim_number'])
        parsed_entry['raw'] = line

        if 'hgnc_symbol' in parsed_entry:
            parsed_entry['hgnc_symbol'] = parsed_entry['hgnc_symbol']

        if parsed_entry.get('entrez_gene_id'):
            parsed_entry['entrez_gene_id'] = int(parsed_entry['entrez_gene_id'])

        if parsed_entry.get('ensembl_gene_id'):
            ensembl_info = parsed_entry['ensembl_gene_id'].split(',')
            parsed_entry['ensembl_gene_id'] = ensembl_info[0].strip()
            if len(ensembl_info) > 1:
                parsed_entry['ensembl_transcript_id'] = ensembl_info[1].strip()

        yield parsed_entry


Instruction:
How would you explain what the following Python 3 function does
def parse_mim_titles(lines):
    header = ['prefix', 'mim_number', 'preferred_title', 'alternative_title', 'included_title']
    for i,line in enumerate(lines):
        line = line.rstrip()
        if not line.startswith('#'):
            parsed_entry = parse_omim_line(line, header)
            parsed_entry['mim_number'] = int(parsed_entry['mim_number'])
            parsed_entry['preferred_title'] = parsed_entry['preferred_title'].split(';')[0]
            yield parsed_entry

Response: Parse the mimTitles. txt file


Instruction:
Write a Python 3 script to
get a dictionary with genes and their omim information.

Response:
def get_mim_genes(genemap_lines, mim2gene_lines):
    """Get a dictionary with genes and their omim information

    Args:
        genemap_lines(iterable(str))
        mim2gene_lines(iterable(str))

    Returns.
        hgnc_genes(dict): A dictionary with hgnc_symbol as keys

    """
    LOG.info("Get the mim genes")

    genes = {}
    hgnc_genes = {}

    gene_nr = 0
    no_hgnc = 0

    for entry in parse_mim2gene(mim2gene_lines):
        if 'gene' in entry['entry_type']:
            mim_nr = entry['mim_number']
            gene_nr += 1
            if not 'hgnc_symbol' in entry:
                no_hgnc += 1
            else:
                genes[mim_nr] = entry

    LOG.info("Number of genes without hgnc symbol %s", str(no_hgnc))

    for entry in parse_genemap2(genemap_lines):
        mim_number = entry['mim_number']
        inheritance = entry['inheritance']
        phenotype_info = entry['phenotypes']
        hgnc_symbol = entry['hgnc_symbol']
        hgnc_symbols = entry['hgnc_symbols']
        if mim_number in genes:
            genes[mim_number]['inheritance'] = inheritance
            genes[mim_number]['phenotypes'] = phenotype_info
            genes[mim_number]['hgnc_symbols'] = hgnc_symbols

    for mim_nr in genes:
        gene_info = genes[mim_nr]
        hgnc_symbol = gene_info['hgnc_symbol']

        if hgnc_symbol in hgnc_genes:
            existing_info = hgnc_genes[hgnc_symbol]
            if not existing_info['phenotypes']:
                hgnc_genes[hgnc_symbol] = gene_info

        else:
            hgnc_genes[hgnc_symbol] = gene_info

    return hgnc_genes


Instruction:
Implement a function in Python 3 to
get a dictionary with phenotype numbers as keys and phenotype information as values.

Response:
def get_mim_phenotypes(genemap_lines):
    """Get a dictionary with phenotypes

    Use the mim numbers for phenotypes as keys and phenotype information as
    values.

    Args:
        genemap_lines(iterable(str))

    Returns:
        phenotypes_found(dict): A dictionary with mim_numbers as keys and
        dictionaries with phenotype information as values.

        {
             'description': str, # Description of the phenotype
             'hgnc_symbols': set(), # Associated hgnc symbols
             'inheritance': set(),  # Associated phenotypes
             'mim_number': int, # mim number of phenotype
        }
    """
    # Set with all omim numbers that are phenotypes
    # Parsed from mim2gene.txt
    phenotype_mims = set()

    phenotypes_found = {}

    # Genemap is a file with one entry per gene.
    # Each line hold a lot of information and in specific it
    # has information about the phenotypes that a gene is associated with
    # From this source we collect inheritane patterns and what hgnc symbols
    # a phenotype is associated with
    for entry in parse_genemap2(genemap_lines):
        hgnc_symbol = entry['hgnc_symbol']
        for phenotype in entry['phenotypes']:
            mim_nr = phenotype['mim_number']
            if mim_nr in phenotypes_found:
                phenotype_entry = phenotypes_found[mim_nr]
                phenotype_entry['inheritance'] = phenotype_entry['inheritance'].union(phenotype['inheritance'])
                phenotype_entry['hgnc_symbols'].add(hgnc_symbol)
            else:
                phenotype['hgnc_symbols'] = set([hgnc_symbol])
                phenotypes_found[mim_nr] = phenotype

    return phenotypes_found


Instruction:
Implement a function in Python 3 to
parse the omim files

Response:
def cli(context, morbid, genemap, mim2gene, mim_titles, phenotypes):
    """Parse the omim files"""
    # if not (morbid and genemap and mim2gene, mim_titles):
    #     print("Please provide all files")
    #     context.abort()

    from scout.utils.handle import get_file_handle
    from pprint import pprint as pp

    print("Morbid file: %s" % morbid)
    print("Genemap file: %s" % genemap)
    print("mim2gene file: %s" % mim2gene)
    print("MimTitles file: %s" % mim_titles)

    if morbid:
        morbid_handle = get_file_handle(morbid)
    if genemap:
        genemap_handle = get_file_handle(genemap)
    if mim2gene:
        mim2gene_handle = get_file_handle(mim2gene)
    if mim_titles:
        mimtitles_handle = get_file_handle(mim_titles)

    mim_genes = get_mim_genes(genemap_handle, mim2gene_handle)
    for entry in mim_genes:
        if entry == 'C10orf11':
            pp(mim_genes[entry])

    context.abort()
    if phenotypes:
        if not genemap:
            click.echo("Please provide the genemap file")
            context.abort()
        phenotypes = get_mim_phenotypes(genemap_handle)
        for i,mim_term in enumerate(phenotypes):
            # pp(phenotypes[mim_term])
            pass

    print("Number of phenotypes found: %s" % i)

    context.abort()
    # hgnc_genes = get_mim_genes(genemap_handle, mim2gene_handle)
    # for hgnc_symbol in hgnc_genes:
    #     pp(hgnc_genes[hgnc_symbol])
    # phenotypes = get_mim_phenotypes(genemap_handle, mim2gene_handle, mimtitles_handle)
    # for mim_nr in phenotypes:
    #     pp(phenotypes[mim_nr])

    genes = get_mim_genes(genemap_handle, mim2gene_handle)
    for hgnc_symbol in genes:
        if hgnc_symbol == 'OPA1':
            print(genes[hgnc_symbol])


Instruction:
Can you generate a brief explanation for the following Python 3 code
def convert_number(string):
    res = None
    if isint(string):
        res = int(string)
    elif isfloat(string):
        res = float(string)
    return res

Response: Convert a string to number if int convert to int otherwise float


Instruction:
Can you create a Python 3 function that
updates a single case in the database.

Response:
def case(context, case_id, case_name, institute, collaborator, vcf, vcf_sv,
         vcf_cancer, vcf_research, vcf_sv_research, vcf_cancer_research, peddy_ped,
         reupload_sv, rankscore_treshold, rankmodel_version):
    """
    Update a case in the database
    """
    adapter = context.obj['adapter']
    if not case_id:
        if not (case_name and institute):
            LOG.info("Please specify which case to update.")
            context.abort
        case_id = "{0}-{1}".format(institute, case_name)
    # Check if the case exists
    case_obj = adapter.case(case_id)

    if not case_obj:
        LOG.warning("Case %s could not be found", case_id)
        context.abort()

    case_changed = False
    if collaborator:
        if not adapter.institute(collaborator):
            LOG.warning("Institute %s could not be found", collaborator)
            context.abort()
        if not collaborator in case_obj['collaborators']:
            case_changed = True
            case_obj['collaborators'].append(collaborator)
            LOG.info("Adding collaborator %s", collaborator)

    if vcf:
        LOG.info("Updating 'vcf_snv' to %s", vcf)
        case_obj['vcf_files']['vcf_snv'] = vcf
        case_changed = True
    if vcf_sv:
        LOG.info("Updating 'vcf_sv' to %s", vcf_sv)
        case_obj['vcf_files']['vcf_sv'] = vcf_sv
        case_changed = True
    if vcf_cancer:
        LOG.info("Updating 'vcf_cancer' to %s", vcf_cancer)
        case_obj['vcf_files']['vcf_cancer'] = vcf_cancer
        case_changed = True
    if vcf_research:
        LOG.info("Updating 'vcf_research' to %s", vcf_research)
        case_obj['vcf_files']['vcf_research'] = vcf_research
        case_changed = True
    if vcf_sv_research:
        LOG.info("Updating 'vcf_sv_research' to %s", vcf_sv_research)
        case_obj['vcf_files']['vcf_sv_research'] = vcf_sv_research
        case_changed = True
    if vcf_cancer_research:
        LOG.info("Updating 'vcf_cancer_research' to %s", vcf_cancer_research)
        case_obj['vcf_files']['vcf_cancer_research'] = vcf_cancer_research
        case_changed = True

    if case_changed:
        adapter.update_case(case_obj)

    if reupload_sv:
        LOG.info("Set needs_check to True for case %s", case_id)
        updates = {'needs_check': True}
        if rankscore_treshold:
            updates['sv_rank_model_version'] = rankmodel_version
        if vcf_sv:
            updates['vcf_files.vcf_sv'] = vcf_sv
        if vcf_sv:
            updates['vcf_files.vcf_sv_research'] = vcf_sv_research

        updated_case = adapter.case_collection.find_one_and_update(
            {'_id':case_id},
            {'$set': updates
            },
            return_document=pymongo.ReturnDocument.AFTER
        )
        rankscore_treshold = rankscore_treshold or updated_case.get("rank_score_threshold", 5)
        # Delete and reload the clinical SV variants
        if updated_case['vcf_files'].get('vcf_sv'):
            adapter.delete_variants(case_id, variant_type='clinical', category='sv')
            adapter.load_variants(updated_case, variant_type='clinical',
                                  category='sv', rank_threshold=rankscore_treshold)
        # Delete and reload research SV variants
        if updated_case['vcf_files'].get('vcf_sv_research'):
            adapter.delete_variants(case_id, variant_type='research', category='sv')
            if updated_case.get('is_research'):
                adapter.load_variants(updated_case, variant_type='research',
                                      category='sv', rank_threshold=rankscore_treshold)


Instruction:
Write a Python 3 script for
exporting all transcripts from the database

Response:
def export_transcripts(adapter, build='37'):
    """Export all transcripts from the database

    Args:
        adapter(scout.adapter.MongoAdapter)
        build(str)

    Yields:
        transcript(scout.models.Transcript)
    """
    LOG.info("Exporting all transcripts")

    for tx_obj in adapter.transcripts(build=build):
        yield tx_obj


Instruction:
Can you implement a function in Python 3 that
returns a formatted month as a table.

Response:
def formatmonth(self, theyear, themonth, withyear=True, net=None, qs=None, template='happenings/partials/calendar/month_table.html'):
        """Return a formatted month as a table."""
        context = self.get_context()
        context['month_start_date'] = date(self.yr, self.mo, 1)
        context['week_rows'] = []
        for week in self.monthdays2calendar(theyear, themonth):
            week_row = []
            for day, weekday in week:
                week_row.append(self.formatday(day, weekday))
            context['week_rows'].append(week_row)

        nxt, prev = get_next_and_prev(net)
        extra_qs = ('&' + '&'.join(qs)) if qs else ''
        context['prev_qs'] = mark_safe('?cal_prev=%d%s' % (prev, extra_qs))
        context['next_qs'] = mark_safe('?cal_next=%d%s' % (nxt, extra_qs))
        context['withyear'] = withyear
        return render_to_string(template, context)


Instruction:
Create a Python 3 function for
returning a day as a table cell.

Response:
def formatday(
            self, day, weekday,
            day_template='happenings/partials/calendar/day_cell.html',
            noday_template='happenings/partials/calendar/day_noday_cell.html',
            popover_template='happenings/partials/calendar/popover.html',
            ):
        """Return a day as a table cell."""
        super(EventCalendar, self).formatday(day, weekday)
        now = get_now()
        context = self.get_context()
        context['events'] = []
        context['day'] = day
        context['day_url'] = self.get_day_url(day)
        context['month_start_date'] = date(self.yr, self.mo, 1)
        context['weekday'] = weekday
        context['cssclass'] = self.cssclasses[weekday]
        context['popover_template'] = popover_template
        context['num_events'] = len(self.count.get(day, [])),
        try:
            processed_date = date(self.yr, self.mo, day)
        except ValueError:
            # day is out of range for month
            processed_date = None

        context['month_start_date'] = date(self.yr, self.mo, 1)

        if day == 0:
            template = noday_template
        else:
            template = day_template

        if now.date() == processed_date:
            context['is_current_day'] = True

        if processed_date and (day in self.count):
            for item in self.count[day]:
                self.pk = item[1]
                self.title = item[0]
                for event in self.events:
                    if event.pk == self.pk:
                        event.check_if_cancelled(processed_date)
                        # allow to use event.last_check_if_cancelled and populate event.title.extra

                        context['events'].append(event)

        return render_to_string(template, context)


Instruction:
Write a Python 3 script for
returning a day as a table cell.

Response:
def formatday(self, day, weekday):
        """Return a day as a table cell."""
        return super(MiniEventCalendar, self).formatday(
            day, weekday,
            day_template='happenings/partials/calendar/mini_day_cell.html',
            popover_template='happenings/partials/calendar/mini_popover.html',
        )


Instruction:
How would you code a function in Python 3 to
set some commonly used variables.

Response:
def formatday(self, day, weekday):
        """Set some commonly used variables."""
        self.wkday_not_today = '
' % (
            self.cssclasses[weekday])

        self.wkday_today = (
            '
' % (
                self.cssclasses[weekday])
        )
        if URLS_NAMESPACE:
            url_name = '%s:day_list' % (URLS_NAMESPACE)
        else:
            url_name = 'day_list'

        self.day_url = reverse(url_name, args=(self.yr, self.mo, day))
        self.day = day

        self.anch = '%d' % (
            self.day_url, day
        )

        self.end = '
	'


Instruction:
Can you generate the documentation for the following Python 3 function
def formatmonthname(self, theyear, themonth, withyear=True):
        display_month = month_name[themonth]

        if isinstance(display_month, six.binary_type) and self.encoding:
            display_month = display_month.decode(self.encoding)

        if withyear:
            s = u'%s %s' % (display_month, theyear)
        else:
            s = u'%s' % display_month
        return ('
'
                '<button id="cal-today-btn" class="btn btn-small">'
                'Today</button> %s
' % s)

Response: Return a month name as a table row.


Instruction:
Create a Python 3 function for
populating variables used to build popovers.

Response:
def popover_helper(self):
        """Populate variables used to build popovers."""
        # when
        display_month = month_name[self.mo]

        if isinstance(display_month, six.binary_type) and self.encoding:
            display_month = display_month.decode('utf-8')

        self.when = ('
When: ' + display_month + ' ' +
                     str(self.day) + ', ' + self.event.l_start_date.strftime(
                         LEGACY_CALENDAR_TIME_FORMAT).lstrip('0') + ' - ' +
                     self.event.l_end_date.strftime(LEGACY_CALENDAR_TIME_FORMAT).lstrip('0') +
                     '
')

        if self.event.location.exists():  # where
            self.where = '
Where: '
            for l in self.event.location.all():
                self.where += l.name
            self.where += '
'
        else:
            self.where = ''

        # description
        self.desc = '
Description: ' + self.event.description[:100]
        self.desc += ('...
' if len(self.event.description) > 100
                      else '
')

        self.event_url = self.event.get_absolute_url()  # url
        t = LEGACY_CALENDAR_TIME_FORMAT if self.event.l_start_date.minute else LEGACY_CALENDAR_HOUR_FORMAT
        self.title2 = (self.event.l_start_date.strftime(t).lstrip('0') +
                       ' ' + self.title)


Instruction:
Make a summary of the following Python 3 code
def formatday(self, day, weekday):
        super(EventCalendar, self).formatday(day, weekday)
        now = get_now()
        self.day = day
        out = ''

        if day == 0:
            return ' 	'  # day outside month
        elif now.month == self.mo and now.year == self.yr and day == now.day:
            if day in self.count:
                # don't return just yet
                out = self.wkday_today + self.anch
            else:
                return self.wkday_today + self.anch + self.end
        elif day in self.count:
            # don't return just yet
            out = self.wkday_not_today + self.anch
        else:
            return self.wkday_not_today + self.anch + self.end

        detail = "%s%s%s
View details"
        extras = ('
')
        common = ' style=background:%s;color:%s;'

        # inject style and extras into calendar html
        for item in self.count[day]:
            self.pk = item[1]
            self.title = item[0]
            for event in self.events:
                if event.pk == self.pk:
                    self.event = event
                    self.check_if_cancelled()
                    # self.add_occurrence
                    self.popover_helper()
                    bg, fnt = self.event.get_colors()
            out += ('' +
                    extras % (
                        self.title,
                        detail % (
                            self.when, self.where, self.desc, self.event_url
                        ),
                        common % (bg, fnt)
                    ) +
                    self.title2 + '
')

        return out + self.end

Response: Return a day as a table cell.


Instruction:
Create a Python 3 function to
return a day as a table cell.

Response:
def formatday(self, day, weekday):
        """Return a day as a table cell."""
        super(MiniEventCalendar, self).formatday(day, weekday)
        now = get_now()
        self.day = day

        if day == 0:
            return ' 	'  # day outside month
        elif now.month == self.mo and now.year == self.yr and day == now.day:
            if day in self.count:
                self.popover_helper()
                return self.wkday_today + self.anch + self.cal_event + self.end
            else:
                return self.wkday_today + self.anch + self.end
        elif day in self.count:
            self.popover_helper()
            return self.wkday_not_today + self.anch + self.cal_event + self.end
        else:
            return self.wkday_not_today + self.anch + self.end


Instruction:
How would you explain what the following Python 3 function does
def get_panel_info(panel_lines=None, panel_id=None, institute=None, version=None, date=None,
                   display_name=None):
    """Parse metadata for a gene panel

    For historical reasons it is possible to include all information about a gene panel in the
    header of a panel file. This function parses the header.

    Args:
        panel_lines(iterable(str))

    Returns:
        panel_info(dict): Dictionary with panel information
    """
    panel_info = {
        'panel_id': panel_id,
        'institute': institute,
        'version': version,
        'date': date,
        'display_name': display_name,
    }

    if panel_lines:
        for line in panel_lines:
            line = line.rstrip()
            if not line.startswith('##'):
                break

            info = line[2:].split('=')
            field = info[0]
            value = info[1]

            if not panel_info.get(field):
                panel_info[field] = value

    panel_info['date'] = get_date(panel_info['date'])

    return panel_info

Response: Parse the panel metadata for a gene panel.


Instruction:
Write a Python 3 script to
parse a gene line with information from a panel file containing a set of related objects.

Response:
def parse_gene(gene_info):
    """Parse a gene line with information from a panel file

        Args:
            gene_info(dict): dictionary with gene info

        Returns:
            gene(dict): A dictionary with the gene information
                {
                'hgnc_id': int,
                'hgnc_symbol': str,
                'disease_associated_transcripts': list(str),
                'inheritance_models': list(str),
                'mosaicism': bool,
                'reduced_penetrance': bool,
                'database_entry_version': str,
                }

    """
    gene = {}
    # This is either hgnc id or hgnc symbol
    identifier = None

    hgnc_id = None
    try:
        if 'hgnc_id' in gene_info:
            hgnc_id = int(gene_info['hgnc_id'])
        elif 'hgnc_idnumber' in gene_info:
            hgnc_id = int(gene_info['hgnc_idnumber'])
        elif 'hgncid' in gene_info:
            hgnc_id = int(gene_info['hgncid'])
    except ValueError as e:
        raise SyntaxError("Invalid hgnc id: {0}".format(hgnc_id))

    gene['hgnc_id'] = hgnc_id
    identifier = hgnc_id

    hgnc_symbol = None
    if 'hgnc_symbol' in gene_info:
        hgnc_symbol = gene_info['hgnc_symbol']
    elif 'hgncsymbol' in gene_info:
        hgnc_symbol = gene_info['hgncsymbol']
    elif 'symbol' in gene_info:
        hgnc_symbol = gene_info['symbol']

    gene['hgnc_symbol'] = hgnc_symbol

    if not identifier:
        if hgnc_symbol:
            identifier = hgnc_symbol
        else:
            raise SyntaxError("No gene identifier could be found")
    gene['identifier'] = identifier
    # Disease associated transcripts is a ','-separated list of
    # manually curated transcripts
    transcripts = ""
    if 'disease_associated_transcripts' in gene_info:
        transcripts = gene_info['disease_associated_transcripts']
    elif 'disease_associated_transcript' in gene_info:
        transcripts = gene_info['disease_associated_transcript']
    elif 'transcripts' in gene_info:
        transcripts = gene_info['transcripts']

    gene['transcripts'] = [
            transcript.strip() for transcript in
            transcripts.split(',') if transcript
        ]

    # Genetic disease models is a ','-separated list of manually curated
    # inheritance patterns that are followed for a gene
    models = ""
    if 'genetic_disease_models' in gene_info:
        models = gene_info['genetic_disease_models']
    elif 'genetic_disease_model' in gene_info:
        models = gene_info['genetic_disease_model']
    elif 'inheritance_models' in gene_info:
        models = gene_info['inheritance_models']
    elif 'genetic_inheritance_models' in gene_info:
        models = gene_info['genetic_inheritance_models']

    gene['inheritance_models'] = [
        model.strip() for model in models.split(',')
        if model.strip() in VALID_MODELS
    ]

    # If a gene is known to be associated with mosaicism this is annotated
    gene['mosaicism'] = True if gene_info.get('mosaicism') else False

    # If a gene is known to have reduced penetrance this is annotated
    gene['reduced_penetrance'] = True if gene_info.get('reduced_penetrance') else False

    # The database entry version is a way to track when a a gene was added or
    # modified, optional
    gene['database_entry_version'] = gene_info.get('database_entry_version')

    return gene


Instruction:
Can you generate the documentation for the following Python 3 function
def parse_genes(gene_lines):
    genes = []
    header = []
    hgnc_identifiers = set()
    delimiter = '\t'
    # This can be '\t' or ';'
    delimiters = ['\t', ' ', ';']

    # There are files that have '#' to indicate headers
    # There are some files that start with a header line without
    # any special symbol
    for i,line in enumerate(gene_lines):
        line = line.rstrip()
        if not len(line) > 0:
            continue
        if line.startswith('#'):
            if not line.startswith('##'):
                # We need to try delimiters
                # We prefer ';' or '\t' byt should accept ' '
                line_length = 0
                delimiter = None
                for alt in delimiters:
                    head_line = line.split(alt)
                    if len(head_line) > line_length:
                        line_length = len(head_line)
                        delimiter = alt

                header = [word.lower() for word in line[1:].split(delimiter)]
        else:
            # If no header symbol(#) assume first line is header
            if i == 0:
                line_length = 0
                for alt in delimiters:
                    head_line = line.split(alt)
                    if len(head_line) > line_length:
                        line_length = len(head_line)
                        delimiter = alt

                if ('hgnc' in line or 'HGNC' in line):
                    header = [word.lower() for word in line.split(delimiter)]
                    continue
                # If first line is not a header try to sniff what the first
                # columns holds
                if line.split(delimiter)[0].isdigit():
                    header = ['hgnc_id']
                else:
                    header = ['hgnc_symbol']

            splitted_line = line.split(delimiter)
            gene_info = dict(zip(header, splitted_line))

            # There are cases when excel exports empty lines that looks like
            # ;;;;;;;. This is a exception to handle these
            info_found = False
            for key in gene_info:
                if gene_info[key]:
                    info_found = True
                    break
            # If no info was found we skip that line
            if not info_found:
                continue

            try:
                gene = parse_gene(gene_info)
            except Exception as e:
                LOG.warning(e)
                raise SyntaxError("Line {0} is malformed".format(i + 1))

            identifier = gene.pop('identifier')

            if not identifier in hgnc_identifiers:
                hgnc_identifiers.add(identifier)
                genes.append(gene)

    return genes

Response: Parse a file with genes and return the hgnc ids


Instruction:
How would you implement a function in Python 3 that
parses the panel info and return a dictionary of the correct format

Response:
def parse_gene_panel(path, institute='cust000', panel_id='test', panel_type='clinical', date=datetime.now(),
                     version=1.0, display_name=None, genes = None):
    """Parse the panel info and return a gene panel

        Args:
            path(str): Path to panel file
            institute(str): Name of institute that owns the panel
            panel_id(str): Panel id
            date(datetime.datetime): Date of creation
            version(float)
            full_name(str): Option to have a long name

        Returns:
            gene_panel(dict)
    """
    LOG.info("Parsing gene panel %s", panel_id)
    gene_panel = {}

    gene_panel['path'] = path
    gene_panel['type'] = panel_type
    gene_panel['date'] = date
    gene_panel['panel_id'] = panel_id
    gene_panel['institute'] = institute
    version = version or 1.0
    gene_panel['version'] = float(version)
    gene_panel['display_name'] = display_name or panel_id

    if not path:
        panel_handle = genes
    else:
        panel_handle = get_file_handle(gene_panel['path'])
    gene_panel['genes'] = parse_genes(gene_lines=panel_handle)

    return gene_panel


Instruction:
Implement a Python 3 function for
parsing a panel app gene into a set of dicts.

Response:
def parse_panel_app_gene(app_gene, hgnc_map):
    """Parse a panel app formated gene

    Args:
        app_gene(dict): Dict with panel app info
        hgnc_map(dict): Map from hgnc_symbol to hgnc_id

    Returns:
        gene_info(dict): Scout infromation
    """
    gene_info = {}
    confidence_level = app_gene['LevelOfConfidence']
    # Return empty gene if not confident gene
    if not confidence_level == 'HighEvidence':
        return gene_info

    hgnc_symbol = app_gene['GeneSymbol']
    # Returns a set of hgnc ids
    hgnc_ids = get_correct_ids(hgnc_symbol, hgnc_map)
    if not hgnc_ids:
        LOG.warning("Gene %s does not exist in database. Skipping gene...", hgnc_symbol)
        return gene_info

    if len(hgnc_ids) > 1:
        LOG.warning("Gene %s has unclear identifier. Choose random id", hgnc_symbol)

    gene_info['hgnc_symbol'] = hgnc_symbol
    for hgnc_id in hgnc_ids:
        gene_info['hgnc_id'] = hgnc_id

    gene_info['reduced_penetrance'] = INCOMPLETE_PENETRANCE_MAP.get(app_gene['Penetrance'])

    inheritance_models = []
    for model in MODELS_MAP.get(app_gene['ModeOfInheritance'],[]):
        inheritance_models.append(model)

    gene_info['inheritance_models'] = inheritance_models

    return gene_info


Instruction:
Explain what the following Python 3 code does
def parse_panel_app_panel(panel_info, hgnc_map, institute='cust000', panel_type='clinical'):
    date_format = "%Y-%m-%dT%H:%M:%S.%f"

    gene_panel = {}
    gene_panel['version'] = float(panel_info['version'])
    gene_panel['date'] = get_date(panel_info['Created'][:-1], date_format=date_format)
    gene_panel['display_name'] = panel_info['SpecificDiseaseName']
    gene_panel['institute'] = institute
    gene_panel['panel_type'] = panel_type

    LOG.info("Parsing panel %s", gene_panel['display_name'])

    gene_panel['genes'] = []

    nr_low_confidence = 1
    nr_genes = 0
    for nr_genes, gene in enumerate(panel_info['Genes'],1):
        gene_info = parse_panel_app_gene(gene, hgnc_map)
        if not gene_info:
            nr_low_confidence += 1
            continue
        gene_panel['genes'].append(gene_info)

    LOG.info("Number of genes in panel %s", nr_genes)
    LOG.info("Number of low confidence genes in panel %s", nr_low_confidence)

    return gene_panel

Response: Parse a PanelApp panel into a dict.


Instruction:
Create a Python 3 function to
return all genes that should be included in the OMIM - AUTO panel

Response:
def get_omim_panel_genes(genemap2_lines, mim2gene_lines, alias_genes):
    """Return all genes that should be included in the OMIM-AUTO panel
    Return the hgnc symbols

    Genes that have at least one 'established' or 'provisional' phenotype connection
    are included in the gene panel

    Args:
        genemap2_lines(iterable)
        mim2gene_lines(iterable)
        alias_genes(dict): A dictionary that maps hgnc_symbol to hgnc_id

    Yields:
        hgnc_symbol(str)
    """
    parsed_genes = get_mim_genes(genemap2_lines, mim2gene_lines)

    STATUS_TO_ADD = set(['established', 'provisional'])

    for hgnc_symbol in parsed_genes:
        try:
            gene = parsed_genes[hgnc_symbol]
            keep = False
            for phenotype_info in gene.get('phenotypes',[]):
                if phenotype_info['status'] in STATUS_TO_ADD:
                    keep = True
                    break
            if keep:
                hgnc_id_info = alias_genes.get(hgnc_symbol)
                if not hgnc_id_info:
                    for symbol in gene.get('hgnc_symbols', []):
                        if symbol in alias_genes:
                            hgnc_id_info = alias_genes[symbol]
                            break

                if hgnc_id_info:
                    yield {
                    'hgnc_id': hgnc_id_info['true'],
                    'hgnc_symbol': hgnc_symbol,
                    }
                else:
                    LOG.warning("Gene symbol %s does not exist", hgnc_symbol)

        except KeyError:
            pass


Instruction:
Implement a Python 3 function for
showing all diseases in the database

Response:
def diseases(context):
    """Show all diseases in the database"""
    LOG.info("Running scout view diseases")
    adapter = context.obj['adapter']

    disease_objs = adapter.disease_terms()

    nr_diseases = disease_objs.count()
    if nr_diseases == 0:
        click.echo("No diseases found")
    else:
        click.echo("Disease")
        for disease_obj in adapter.disease_terms():
            click.echo("{0}".format(disease_obj['_id']))
        LOG.info("{0} diseases found".format(nr_diseases))


Instruction:
Can you tell what is the following Python 3 function doing
def hpo(context):
    LOG.info("Running scout update hpo")
    adapter = context.obj['adapter']

    LOG.info("Dropping HPO terms")
    adapter.hpo_term_collection.drop()
    LOG.debug("HPO terms dropped")

    load_hpo_terms(adapter)

Response: Update the HPO terms in the database.


Instruction:
Can you generate the documentation for the following Python 3 function
def get_upcoming_events(self):
        if self.event.repeats('NEVER'):
            has_ended = False
            now_gt_start = self.now > self.event.l_start_date
            now_gt_end = self.now > self.event.end_date
            if now_gt_end or now_gt_start:
                has_ended = True
            has_not_started = self.event.l_start_date > self.finish
            if has_ended or has_not_started:
                return self.events
            self.events.append((self.event.l_start_date, self.event))
            return self.events
        if self.event.repeats('WEEKDAY'):
            self._weekday()
        elif self.event.repeats('MONTHLY'):
            self._monthly()
        elif self.event.repeats('YEARLY'):
            self._yearly()
        else:
            self._others()
        return self.events

Response: Repeats an event and returns num or fewer upcoming events from now.


Instruction:
Write a Python 3 script to
check start to see if we should stop collecting upcoming events.

Response:
def we_should_stop(self, start, start_):
        """
        Checks 'start' to see if we should stop collecting upcoming events.
        'start' should be a datetime.datetime, 'start_' should be the same
        as 'start', but it should be a datetime.date to allow comparison
        w/ end_repeat.
        """
        if start > self.finish or \
                self.event.end_repeat is not None and \
                start_ > self.event.end_repeat:
            return True
        else:
            return False


Instruction:
Write a Python 3 script for
displaying a list of all users and which institutes they belong to.

Response:
def users(store):
    """Display a list of all users and which institutes they belong to."""
    user_objs = list(store.users())
    total_events = store.user_events().count()
    for user_obj in user_objs:
        if user_obj.get('institutes'):
            user_obj['institutes'] = [store.institute(inst_id) for inst_id in user_obj.get('institutes')]
        else:
            user_obj['institutes'] = []
        user_obj['events'] = store.user_events(user_obj).count()
        user_obj['events_rank'] = event_rank(user_obj['events'])
    return dict(
        users=sorted(user_objs, key=lambda user: -user['events']),
        total_events=total_events,
    )


Instruction:
Write a Python 3 function that can
parse the conservation predictors for a single variant.

Response:
def parse_conservations(variant):
    """Parse the conservation predictors

        Args:
            variant(dict): A variant dictionary

        Returns:
            conservations(dict): A dictionary with the conservations
    """
    conservations = {}

    conservations['gerp'] = parse_conservation(
                                            variant,
                                            'dbNSFP_GERP___RS'
                                        )
    conservations['phast'] = parse_conservation(
                                            variant,
                                            'dbNSFP_phastCons100way_vertebrate'
                                        )
    conservations['phylop'] = parse_conservation(
                                            variant,
                                            'dbNSFP_phyloP100way_vertebrate'
                                        )
    return conservations


Instruction:
Given the following Python 3 function, write the documentation
def parse_conservation(variant, info_key):
    raw_score = variant.INFO.get(info_key)
    conservations = []

    if raw_score:
        if isinstance(raw_score, numbers.Number):
            raw_score = (raw_score,)

        for score in raw_score:
            if score >= CONSERVATION[info_key]['conserved_min']:
                conservations.append('Conserved')
            else:
                conservations.append('NotConserved')

    return conservations

Response: Get the conservation prediction


Instruction:
Can you tell what is the following Python 3 function doing
def get_next_event(event, now):
    year = now.year
    month = now.month
    day = now.day
    e_day = event[0].l_start_date.day
    e_end_day = event[0].l_end_date.day
    good_today = True if event[0].l_start_date.time() >= now.time() else False
    if event[0].starts_same_year_month_as(year, month) and \
            e_day <= now.day <= e_end_day:
        occurrences = CountHandler(year, month, event).get_count()
        future_dates = (x for x in occurrences if x >= now.day)
        day = min(future_dates, key=lambda x: abs(x - now.day))
    else:
        e_year = event[0].l_start_date.year
        e_month = event[0].l_start_date.month
        # convert to datetime.date() to be sure we can make a comparison
        if date(e_year, e_month, e_day) > date(now.year, now.month, now.day):
            # if the event hasn't started yet, then its next occurrence will
            # be on its start date, so return that.
            year = e_year
            month = e_month
            day = e_day
        else:
            occurrences = CountHandler(year, month, event).get_count()
            future_dates = [x for x in occurrences if x >= now.day]
            e_end_month = event[0].l_end_date.month
            if future_dates and future_dates[0] is day and not good_today:
                future_dates.pop(0)
            while not future_dates:
                month, year = inc_month(month, year)
                if event[0].repeats('YEARLY') and \
                        (month != e_month or month != e_end_month):
                    continue
                occurrences = CountHandler(year, month, event).get_count()
                # we don't check for now.day here, b/c we're in a month past
                # whatever now is. As an example, if we checked for now.day
                # we'd get stuck in an infinite loop if this were a
                # monthly repeating event and our 'now' was on a day after the
                # event's l_end_date.day
                future_dates = [x for x in occurrences]
            day = min(future_dates)

    if event[0].repeats('WEEKDAY'):
        return check_weekday(year, month, day)

    return year, month, day

Response: Returns the next occurrence of a given event relative to now.


Instruction:
Can you write a function in Python 3 where it
returns the informations for a single instance of a base class based on the phenotype of the cases.

Response:
def get_dashboard_info(adapter, institute_id=None, slice_query=None):
    """Returns cases with phenotype

        If phenotypes are provided search for only those

    Args:
        adapter(adapter.MongoAdapter)
        institute_id(str): an institute _id
        slice_query(str):  query to filter cases to obtain statistics for.

    Returns:
        data(dict): Dictionary with relevant information
    """

    LOG.debug("General query with institute_id {}.".format(institute_id))

    # if institute_id == 'None' or None, all cases and general stats will be returned
    if institute_id == 'None':
        institute_id = None

    # If a slice_query is present then numbers in "General statistics" and "Case statistics" will
    # reflect the data available for the query
    general_sliced_info = get_general_case_info(adapter, institute_id=institute_id,
                                                    slice_query=slice_query)
    total_sliced_cases = general_sliced_info['total_cases']

    data = {'total_cases': total_sliced_cases}
    if total_sliced_cases == 0:
        return data

    data['pedigree'] = []
    for ped_info in general_sliced_info['pedigree'].values():
        ped_info['percent'] = ped_info['count'] / total_sliced_cases
        data['pedigree'].append(ped_info)

    data['cases'] = get_case_groups(adapter, total_sliced_cases,
                        institute_id=institute_id, slice_query=slice_query)

    data['analysis_types'] = get_analysis_types(adapter, total_sliced_cases,
                                institute_id=institute_id, slice_query=slice_query)

    overview = [
        {
            'title': 'Phenotype terms',
            'count': general_sliced_info['phenotype_cases'],
            'percent': general_sliced_info['phenotype_cases'] / total_sliced_cases,
        },
        {
            'title': 'Causative variants',
            'count': general_sliced_info['causative_cases'],
            'percent': general_sliced_info['causative_cases'] / total_sliced_cases,
        },
        {
            'title': 'Pinned variants',
            'count': general_sliced_info['pinned_cases'],
            'percent': general_sliced_info['pinned_cases'] / total_sliced_cases,
        },
        {
            'title': 'Cohort tag',
            'count': general_sliced_info['cohort_cases'],
            'percent': general_sliced_info['cohort_cases'] / total_sliced_cases,
        }
    ]

    # Data from "Variant statistics tab" is not filtered by slice_query and numbers will
    # reflect verified variants in all available cases for an institute
    general_info = get_general_case_info(adapter, institute_id=institute_id)
    total_cases = general_info['total_cases']
    sliced_case_ids = general_sliced_info['case_ids']
    verified_query = {
        'verb' : 'validate',
    }
    if institute_id: # filter by institute if users wishes so
        verified_query['institute'] =  institute_id

    # Case level information
    sliced_validation_cases = set()
    sliced_validated_cases = set()

    # Variant level information
    validated_tp = set()
    validated_fp = set()
    var_valid_orders = 0 # use this counter to count 'True Positive', 'False positive' and 'Not validated' vars

    validate_events = adapter.event_collection.find(verified_query)
    for validate_event in list(validate_events):
        case_id = validate_event.get('case')
        var_obj = adapter.variant(case_id=case_id, document_id=validate_event['variant_id'])

        if var_obj: # Don't take into account variants which have been removed from db
            var_valid_orders += 1
            if case_id in sliced_case_ids:
                sliced_validation_cases.add(case_id) # add to the set. Can't add same id twice since it'a a set

            validation = var_obj.get('validation')
            if validation and validation in ['True positive', 'False positive']:
                if case_id in sliced_case_ids:
                    sliced_validated_cases.add(case_id)
                if validation == 'True positive':
                    validated_tp.add(var_obj['_id'])
                elif validation == 'False positive':
                    validated_fp.add(var_obj['_id'])

    n_validation_cases = len(sliced_validation_cases)
    n_validated_cases = len(sliced_validated_cases)

    # append
    overview.append(
    {
        'title': 'Validation ordered',
        'count': n_validation_cases,
        'percent': n_validation_cases / total_sliced_cases,
    })
    overview.append(
    {
        'title': 'Validated cases (TP + FP)',
        'count': n_validated_cases,
        'percent': n_validated_cases / total_sliced_cases,
    })

    data['overview'] = overview

    variants = []
    nr_validated = len(validated_tp) + len(validated_fp)
    variants.append(
        {
            'title': 'Validation ordered',
            'count': var_valid_orders,
            'percent': 1
        }
    )

    # taking into account that var_valid_orders might be 0:
    percent_validated_tp = 0
    percent_validated_fp = 0

    if var_valid_orders:
        percent_validated_tp = len(validated_tp) / var_valid_orders
        percent_validated_fp = len(validated_fp) / var_valid_orders

    variants.append(
        {
            'title': 'Validated True Positive',
            'count': len(validated_tp),
            'percent': percent_validated_tp,
        }
    )

    variants.append(
        {
            'title': 'Validated False Positive',
            'count': len(validated_fp),
            'percent': percent_validated_fp,
        }
    )

    data['variants'] = variants
    return data


Instruction:
Write a Python 3 function for
returning a dictionary of general information about the cases.

Response:
def get_general_case_info(adapter, institute_id=None, slice_query=None):
    """Return general information about cases

    Args:
        adapter(adapter.MongoAdapter)
        institute_id(str)
        slice_query(str):   Query to filter cases to obtain statistics for.

    Returns:
        general(dict)
    """
    general = {}

    # Potentially sensitive slice queries are assumed allowed if we have got this far
    name_query = slice_query

    cases = adapter.cases(owner=institute_id, name_query=name_query)

    phenotype_cases = 0
    causative_cases = 0
    pinned_cases = 0
    cohort_cases = 0

    pedigree = {
        1: {
            'title': 'Single',
            'count': 0
        },
        2: {
            'title': 'Duo',
            'count': 0
        },
        3: {
            'title': 'Trio',
            'count': 0
        },
        'many': {
            'title': 'Many',
            'count': 0
        },
    }

    case_ids = set()

    total_cases = 0
    for total_cases,case in enumerate(cases,1):
        # If only looking at one institute we need to save the case ids
        if institute_id:
            case_ids.add(case['_id'])
        if case.get('phenotype_terms'):
            phenotype_cases += 1
        if case.get('causatives'):
            causative_cases += 1
        if case.get('suspects'):
            pinned_cases += 1
        if case.get('cohorts'):
            cohort_cases += 1

        nr_individuals = len(case.get('individuals',[]))
        if nr_individuals == 0:
            continue
        if nr_individuals > 3:
            pedigree['many']['count'] += 1
        else:
            pedigree[nr_individuals]['count'] += 1

    general['total_cases'] = total_cases
    general['phenotype_cases'] = phenotype_cases
    general['causative_cases'] = causative_cases
    general['pinned_cases'] = pinned_cases
    general['cohort_cases'] = cohort_cases
    general['pedigree'] = pedigree
    general['case_ids'] = case_ids

    return general


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_case_groups(adapter, total_cases, institute_id=None, slice_query=None):
    # Create a group with all cases in the database
    cases = [{'status': 'all', 'count': total_cases, 'percent': 1}]
    # Group the cases based on their status
    pipeline = []
    group = {'$group' : {'_id': '$status', 'count': {'$sum': 1}}}

    subquery = {}
    if institute_id and slice_query:
        subquery = adapter.cases(owner=institute_id, name_query=slice_query,
                              yield_query=True)
    elif institute_id:
        subquery = adapter.cases(owner=institute_id, yield_query=True)
    elif slice_query:
        subquery = adapter.cases(name_query=slice_query, yield_query=True)

    query = {'$match': subquery} if subquery else {}

    if query:
        pipeline.append(query)

    pipeline.append(group)
    res = adapter.case_collection.aggregate(pipeline)

    for status_group in res:
        cases.append({'status': status_group['_id'],
                      'count': status_group['count'],
                      'percent': status_group['count'] / total_cases})

    return cases

Response: Return the information about the cases in the cluster.


Instruction:
How would you code a function in Python 3 to
return information about analysis types.

Response:
def get_analysis_types(adapter, total_cases, institute_id=None, slice_query=None):
    """ Return information about analysis types.
        Group cases based on analysis type for the individuals.
    Args:
        adapter(adapter.MongoAdapter)
        total_cases(int): Total number of cases
        institute_id(str)
        slice_query(str): Query to filter cases to obtain statistics for.
    Returns:
        analysis_types array of hashes with name: analysis_type(str), count: count(int)

    """
    # Group cases based on analysis type of the individuals
    query = {}

    subquery = {}
    if institute_id and slice_query:
        subquery = adapter.cases(owner=institute_id, name_query=slice_query,
                              yield_query=True)
    elif institute_id:
        subquery = adapter.cases(owner=institute_id, yield_query=True)
    elif slice_query:
        subquery = adapter.cases(name_query=slice_query, yield_query=True)

    query = {'$match': subquery}

    pipeline = []
    if query:
        pipeline.append(query)

    pipeline.append({'$unwind': '$individuals'})
    pipeline.append({'$group': {'_id': '$individuals.analysis_type', 'count': {'$sum': 1}}})
    analysis_query = adapter.case_collection.aggregate(pipeline)
    analysis_types = [{'name': group['_id'], 'count': group['count']} for group in analysis_query]

    return analysis_types


Instruction:
Here you have a function in Python 3, explain what it does
def render_to_json_response(self, context, **kwargs):
        return HttpResponse(
            self.convert_context_to_json(context),
            content_type='application/json',
            **kwargs
        )

Response: Returns a JSON response transforming the context to make the payload.


Instruction:
Can you implement a function in Python 3 that
converts the context dict to a JSON object.

Response:
def convert_context_to_json(self, context):
        """
        Get what we want out of the context dict and convert that to a JSON
        object. Note that this does no object serialization b/c we're
        not sending any objects.
        """
        if 'month/shift' in self.request.path:  # month calendar
            return dumps(self.get_month_calendar_dict(context))
        elif 'event-list/shift' in self.request.path:  # month event list
            return dumps(self.get_month_event_list_dict(context))
        elif 'cal-and-list/shift' in self.request.path:
            cal = self.get_month_calendar_dict(context)
            l = self.get_month_event_list_dict(context)
            cal.update(l)
            return dumps(cal)
        else:  # day list view
            for key, val in context.items():
                if isinstance(val, Promise):
                    context[key] = force_text(val)

            return dumps(self.get_day_context_dict(context))


Instruction:
Write a Python 3 function for
getting the year and month from the net and the querystring.

Response:
def get_year_and_month(self, net, qs, **kwargs):
        """
        Get the year and month. First tries from kwargs, then from
        querystrings. If none, or if cal_ignore qs is specified,
        sets year and month to this year and this month.
        """
        now = c.get_now()
        year = now.year
        month = now.month + net
        month_orig = None

        if 'cal_ignore=true' not in qs:
            if 'year' and 'month' in self.kwargs:  # try kwargs
                year, month_orig = map(
                    int, (self.kwargs['year'], self.kwargs['month'])
                )
                month = month_orig + net
            else:
                try:  # try querystring
                    year = int(self.request.GET['cal_year'])
                    month_orig = int(self.request.GET['cal_month'])
                    month = month_orig + net
                except Exception:
                    pass
        # return the year and month, and any errors that may have occurred do
        # to an invalid month/year being given.
        return c.clean_year_month(year, month, month_orig)


Instruction:
Given the following Python 3 function, write the documentation
def check_for_cancelled_events(self, d):
        for event in self.events:
            for cn in event.cancellations.all():
                if cn.date == d:
                    event.title += ' (CANCELLED)'

Response: Check if any events are cancelled on the given date d.


Instruction:
Write a Python 3 function for
loading a HPO term into the database.

Response:
def load_hpo_term(self, hpo_obj):
        """Add a hpo object

        Arguments:
            hpo_obj(dict)

        """
        LOG.debug("Loading hpo term %s into database", hpo_obj['_id'])
        try:
            self.hpo_term_collection.insert_one(hpo_obj)
        except DuplicateKeyError as err:
            raise IntegrityError("Hpo term %s already exists in database".format(hpo_obj['_id']))
        LOG.debug("Hpo term saved")


Instruction:
Can you implement a function in Python 3 that
loads a list of HPO terms into the bulkwrite database

Response:
def load_hpo_bulk(self, hpo_bulk):
        """Add a hpo object

        Arguments:
            hpo_bulk(list(scout.models.HpoTerm))

        Returns:
            result: pymongo bulkwrite result

        """
        LOG.debug("Loading hpo bulk")

        try:
            result = self.hpo_term_collection.insert_many(hpo_bulk)
        except (DuplicateKeyError, BulkWriteError) as err:
            raise IntegrityError(err)
        return result


Instruction:
How would you implement a function in Python 3 that
fetches a hpo term

Response:
def hpo_term(self, hpo_id):
        """Fetch a hpo term

        Args:
            hpo_id(str)

        Returns:
            hpo_obj(dict)
        """
        LOG.debug("Fetching hpo term %s", hpo_id)

        return self.hpo_term_collection.find_one({'_id': hpo_id})


Instruction:
Can you create a Python 3 function that
searches for all HPO terms in the database.

Response:
def hpo_terms(self, query=None, hpo_term=None, text=None, limit=None):
        """Return all HPO terms

        If a query is sent hpo_terms will try to match with regex on term or
        description.

        Args:
            query(str): Part of a hpoterm or description
            hpo_term(str): Search for a specific hpo term
            limit(int): the number of desired results

        Returns:
            result(pymongo.Cursor): A cursor with hpo terms
        """
        query_dict = {}
        search_term = None
        if query:
            query_dict = {'$or':
                [
                    {'hpo_id': {'$regex': query, '$options':'i'}},
                    {'description': {'$regex': query, '$options':'i'}},
                ]
            }
            search_term = query
        elif text:
            new_string = ''
            for i,word in enumerate(text.split(' ')):
                if i == 0:
                    new_string += word
                else:
                    new_string += ' \"{0}\"'.format(word)
            LOG.info("Search HPO terms with %s", new_string)
            query_dict['$text'] = {'$search': new_string}
            search_term = text
        elif hpo_term:
            query_dict['hpo_id'] = hpo_term
            search_term = hpo_term

        limit = limit or int(10e10)
        res = self.hpo_term_collection.find(query_dict).limit(limit).sort('hpo_number',ASCENDING)

        LOG.info("Found {0} terms with search word {1}".format(res.count(), search_term))
        return res


Instruction:
Can you write a function in Python 3 where it
returns a disease term object based on the identifier

Response:
def disease_term(self, disease_identifier):
        """Return a disease term

        Checks if the identifier is a disease number or a id

        Args:
            disease_identifier(str)

        Returns:
            disease_obj(dict)
        """
        query = {}
        try:
            disease_identifier = int(disease_identifier)
            query['disease_nr'] = disease_identifier
        except ValueError:
            query['_id'] = disease_identifier

        return self.disease_term_collection.find_one(query)


Instruction:
Write a Python 3 script for
returning all disease terms that overlaps a gene

Response:
def disease_terms(self, hgnc_id=None):
        """Return all disease terms that overlaps a gene

            If no gene, return all disease terms

        Args:
            hgnc_id(int)

        Returns:
            iterable(dict): A list with all disease terms that match
        """
        query = {}
        if hgnc_id:
            LOG.debug("Fetching all diseases for gene %s", hgnc_id)
            query['genes'] = hgnc_id
        else:
            LOG.info("Fetching all disease terms")

        return list(self.disease_term_collection.find(query))


Instruction:
Write a Python 3 script for
loading a disease term into the database.

Response:
def load_disease_term(self, disease_obj):
        """Load a disease term into the database

        Args:
            disease_obj(dict)
        """
        LOG.debug("Loading disease term %s into database", disease_obj['_id'])
        try:
            self.disease_term_collection.insert_one(disease_obj)
        except DuplicateKeyError as err:
            raise IntegrityError("Disease term %s already exists in database".format(disease_obj['_id']))

        LOG.debug("Disease term saved")


Instruction:
Make a summary of the following Python 3 code
def generate_hpo_gene_list(self, *hpo_terms):
        genes = {}
        for term in hpo_terms:
            hpo_obj = self.hpo_term(term)
            if hpo_obj:
                for hgnc_id in hpo_obj['genes']:
                    if hgnc_id in genes:
                        genes[hgnc_id] += 1
                    else:
                        genes[hgnc_id] = 1
            else:
                LOG.warning("Term %s could not be found", term)

        sorted_genes = sorted(genes.items(), key=operator.itemgetter(1), reverse=True)
        return sorted_genes

Response: Generate a sorted list of namedtuples of hpogenes


Instruction:
Can you generate the documentation for the following Python 3 function
def cmd_tool(args=None):

    from argparse import ArgumentParser

    parser = ArgumentParser(description="Command line utility for reading and plotting filterbank files.")

    parser.add_argument('-p', action='store',  default='ank', dest='what_to_plot', type=str,
                        help='Show: "w" waterfall (freq vs. time) plot; "s" integrated spectrum plot; \
                        "t" for time series; "mm" for spectrum including min max; "k" for kurtosis; \
                        "a" for all available plots and information; and "ank" for all but kurtosis.')
    parser.add_argument('filename', type=str,
                        help='Name of file to read')
    parser.add_argument('-b', action='store', default=None, dest='f_start', type=float,
                        help='Start frequency (begin), in MHz')
    parser.add_argument('-e', action='store', default=None, dest='f_stop', type=float,
                        help='Stop frequency (end), in MHz')
    parser.add_argument('-B', action='store', default=None, dest='t_start', type=int,
                        help='Start integration (begin) ID')
    parser.add_argument('-E', action='store', default=None, dest='t_stop', type=int,
                        help='Stop integration (end) ID')
    parser.add_argument('-i', action='store_true', default=False, dest='info_only',
                        help='Show info only')
    parser.add_argument('-a', action='store_true', default=False, dest='average',
                       help='average along time axis (plot spectrum only)')
    parser.add_argument('-s', action='store', default='', dest='plt_filename', type=str,
                       help='save plot graphic to file (give filename as argument)')
    parser.add_argument('-S', action='store_true', default=False, dest='save_only',
                       help='Turn off plotting of data and only save to file.')
    parser.add_argument('-D', action='store_false', default=True, dest='blank_dc',
                       help='Use to not blank DC bin.')
    parser.add_argument('-c', action='store_true', default=False, dest='calibrate_band_pass',
                        help='Calibrate band pass.')
    args = parser.parse_args()

    # Open blimpy data
    filename = args.filename
    load_data = not args.info_only

    # only load one integration if looking at spectrum
    wtp = args.what_to_plot
    if not wtp or 's' in wtp:
        if args.t_start == None:
            t_start = 0
        else:
            t_start = args.t_start
        t_stop  = t_start + 1

        if args.average:
            t_start = None
            t_stop  = None
    else:
        t_start = args.t_start
        t_stop  = args.t_stop

    if args.info_only:
        args.blank_dc = False
        args.calibrate_band_pass = False

    fil = Filterbank(filename, f_start=args.f_start, f_stop=args.f_stop,
                     t_start=t_start, t_stop=t_stop,
                     load_data=load_data,blank_dc=args.blank_dc,
                     cal_band_pass=args.calibrate_band_pass)
    fil.info()

    # And if we want to plot data, then plot data.
    if not args.info_only:
        # check start & stop frequencies make sense
        #try:
        #    if args.f_start:
        #        print "Start freq: %2.2f" % args.f_start
        #        assert args.f_start >= fil.freqs[0] or np.isclose(args.f_start, fil.freqs[0])
        #
        #    if args.f_stop:
        #        print "Stop freq: %2.2f" % args.f_stop
        #        assert args.f_stop <= fil.freqs[-1] or np.isclose(args.f_stop, fil.freqs[-1])
        #except AssertionError:
        #    print "Error: Start and stop frequencies must lie inside file's frequency range."
        #    print "i.e. between %2.2f-%2.2f MHz." % (fil.freqs[0], fil.freqs[-1])
        #    exit()

        if args.what_to_plot == "w":
            plt.figure("waterfall", figsize=(8, 6))
            fil.plot_waterfall(f_start=args.f_start, f_stop=args.f_stop)
        elif args.what_to_plot == "s":
            plt.figure("Spectrum", figsize=(8, 6))
            fil.plot_spectrum(logged=True, f_start=args.f_start, f_stop=args.f_stop, t='all')
        elif args.what_to_plot == "mm":
            plt.figure("min max", figsize=(8, 6))
            fil.plot_spectrum_min_max(logged=True, f_start=args.f_start, f_stop=args.f_stop, t='all')
        elif args.what_to_plot == "k":
            plt.figure("kurtosis", figsize=(8, 6))
            fil.plot_kurtosis(f_start=args.f_start, f_stop=args.f_stop)
        elif args.what_to_plot == "t":
            plt.figure("Time Series", figsize=(8, 6))
            fil.plot_time_series(f_start=args.f_start, f_stop=args.f_stop,orientation='h')
        elif args.what_to_plot == "a":
            plt.figure("Multiple diagnostic plots", figsize=(12, 9),facecolor='white')
            fil.plot_all(logged=True, f_start=args.f_start, f_stop=args.f_stop, t='all')
        elif args.what_to_plot == "ank":
            plt.figure("Multiple diagnostic plots", figsize=(12, 9),facecolor='white')
            fil.plot_all(logged=True, f_start=args.f_start, f_stop=args.f_stop, t='all',kurtosis=False)

        if args.plt_filename != '':
            plt.savefig(args.plt_filename)

        if not args.save_only:
            if 'DISPLAY' in os.environ.keys():
                plt.show()
            else:
                print("No $DISPLAY available.")

Response: Command line tool for reading and plotting filterbank files.


Instruction:
Given the following Python 3 function, write the documentation
def read_hdf5(self, filename, f_start=None, f_stop=None,
                        t_start=None, t_stop=None, load_data=True):
        """ Populate Filterbank instance with data from HDF5 file

        Note:
            This is to be deprecated in future, please use Waterfall() to open files.
        """
        print("Warning: this function will be deprecated in the future. Please use Waterfall to open HDF5 files.")
#        raise DeprecationWarning('Please use Waterfall to open HDF5 files.')

        self.header = {}
        self.filename = filename
        self.h5 = h5py.File(filename)
        for key, val in self.h5[b'data'].attrs.items():
            if six.PY3:
                key = bytes(key, 'ascii')
            if key == b'src_raj':
                self.header[key] = Angle(val, unit='hr')
            elif key == b'src_dej':
                self.header[key] = Angle(val, unit='deg')
            else:
                self.header[key] = val

        self.n_ints_in_file = self.h5[b"data"].shape[0]
        i_start, i_stop, chan_start_idx, chan_stop_idx = self._setup_freqs(f_start=f_start, f_stop=f_stop)
        ii_start, ii_stop, n_ints = self._setup_time_axis(t_start=t_start, t_stop=t_stop)

        if load_data:
            self.data = self.h5[b"data"][ii_start:ii_stop, :, chan_start_idx:chan_stop_idx]

            self.file_size_bytes = os.path.getsize(self.filename)

#         if self.header[b'foff'] < 0:
#             self.data = self.data[..., ::-1] # Reverse data

        else:
            print("Skipping data load...")
            self.data = np.array([0])
            self.n_ints_in_file  = 0
            self.file_size_bytes = os.path.getsize(self.filename)

Response: Read filterbank from HDF5 file.


Instruction:
Explain what the following Python 3 code does
def read_filterbank(self, filename=None, f_start=None, f_stop=None,
                        t_start=None, t_stop=None, load_data=True):
        """ Populate Filterbank instance with data from Filterbank file

        Note:
            This is to be deprecated in future, please use Waterfall() to open files.
        """
        if filename is None:
            filename = self.filename
        else:
            self.filename = filename

        self.header = read_header(filename)

        #convert input frequencies into what their corresponding index would be
        i_start, i_stop, chan_start_idx, chan_stop_idx = self._setup_freqs(f_start=f_start, f_stop=f_stop)

        n_bits  = self.header[b'nbits']
        n_bytes  = int(self.header[b'nbits'] / 8)
        n_chans = self.header[b'nchans']
        n_chans_selected = self.freqs.shape[0]
        n_ifs   = self.header[b'nifs']

        # Load binary data
        self.idx_data = len_header(filename)
        f = open(filename, 'rb')
        f.seek(self.idx_data)
        filesize = os.path.getsize(self.filename)
        n_bytes_data = filesize - self.idx_data

        # Finally add some other info to the class as objects
        self.n_ints_in_file  = calc_n_ints_in_file(self.filename)
        self.file_size_bytes = filesize

        ## Setup time axis
        ii_start, ii_stop, n_ints = self._setup_time_axis(t_start=t_start, t_stop=t_stop)

        # Seek to first integration
        f.seek(int(ii_start * n_bits * n_ifs * n_chans / 8), 1)

        # Set up indexes used in file read (taken out of loop for speed)
        i0 = np.min((chan_start_idx, chan_stop_idx))
        i1 = np.max((chan_start_idx, chan_stop_idx))

        #Set up the data type (taken out of loop for speed)
        if n_bits == 2:
            dd_type = b'uint8'
            n_chans_selected = int(n_chans_selected/4)
        elif n_bytes == 4:
            dd_type = b'float32'
        elif n_bytes == 2:
            dd_type = b'uint16'
        elif n_bytes == 1:
            dd_type = b'uint8'

        if load_data:

            if n_ints * n_ifs * n_chans_selected > MAX_DATA_ARRAY_SIZE:
                print("[Filterbank]  Error: data array is too large to load. Either select fewer points or manually increase MAX_DATA_ARRAY_SIZE. Large files are now handle with Waterfall .")
                sys.exit()

            if n_bits == 2:
                self.data = np.zeros((n_ints, n_ifs, n_chans_selected*4), dtype=dd_type)
            else:
                self.data = np.zeros((n_ints, n_ifs, n_chans_selected), dtype=dd_type)

            for ii in range(n_ints):
                """d = f.read(n_bytes * n_chans * n_ifs)
                """

                for jj in range(n_ifs):

                    f.seek(n_bytes * i0, 1) # 1 = from current location
                    #d = f.read(n_bytes * n_chans_selected)
                    #bytes_to_read = n_bytes * n_chans_selected

                    dd = np.fromfile(f, count=n_chans_selected, dtype=dd_type)

                    # Reverse array if frequency axis is flipped
#                     if f_delt < 0:
#                         dd = dd[::-1]

                    if n_bits == 2:
                        dd = unpack_2to8(dd)
                    self.data[ii, jj] = dd

                    f.seek(n_bytes * (n_chans - i1), 1)  # Seek to start of next block
        else:
            print("Skipping data load...")
            self.data = np.array([0], dtype=dd_type)

Response: Read a filterbank file and populate the object with data from it.


Instruction:
Can you create a Python 3 function that
computes LST for observation

Response:
def compute_lst(self):
        """ Compute LST for observation """
        if self.header[b'telescope_id'] == 6:
            self.coords = gbt_coords
        elif self.header[b'telescope_id'] == 4:
            self.coords = parkes_coords
        else:
            raise RuntimeError("Currently only Parkes and GBT supported")
        if HAS_SLALIB:
            # dut1 = (0.2 /3600.0) * np.pi/12.0
            dut1 = 0.0
            mjd = self.header[b'tstart']
            tellong = np.deg2rad(self.coords[1])
            last = s.sla_gmst(mjd) - tellong + s.sla_eqeqx(mjd) + dut1
            # lmst = s.sla_gmst(mjd) - tellong
            if last < 0.0 : last = last + 2.0*np.pi
            return last
        else:
            raise RuntimeError("This method requires pySLALIB")


Instruction:
Write a Python 3 script to
compute the LSR in km and s .

Response:
def compute_lsrk(self):
        """ Computes the LSR in km/s

        uses the MJD, RA and DEC of observation to compute
        along with the telescope location. Requires pyslalib
        """
        ra = Angle(self.header[b'src_raj'], unit='hourangle')
        dec = Angle(self.header[b'src_dej'], unit='degree')
        mjdd = self.header[b'tstart']
        rarad = ra.to('radian').value
        dcrad = dec.to('radian').value
        last = self.compute_lst()
        tellat  = np.deg2rad(self.coords[0])
        tellong = np.deg2rad(self.coords[1])

        # convert star position to vector
        starvect = s.sla_dcs2c(rarad, dcrad)

        # velocity component in ra,dec due to Earth rotation
        Rgeo = s.sla_rverot( tellat, rarad, dcrad, last)

        # get Barycentric and heliocentric velocity and position of the Earth.
        evp = s.sla_evp(mjdd, 2000.0)
        dvb = evp[0]   # barycentric velocity vector, in AU/sec
        dpb = evp[1]   # barycentric position vector, in AU
        dvh = evp[2]   # heliocentric velocity vector, in AU/sec
        dph = evp[3]   # heliocentric position vector, in AU

        # dot product of vector to object and heliocentric velocity
        # convert AU/sec to km/sec
        vcorhelio = -s.sla_dvdv( starvect, dvh) *149.597870e6
        vcorbary  = -s.sla_dvdv( starvect, dvb) *149.597870e6

        # rvlsrd is velocity component in ra,dec direction due to the Sun's
        # motion with respect to the "dynamical" local standard of rest
        rvlsrd = s.sla_rvlsrd( rarad,dcrad)

        # rvlsrk is velocity component in ra,dec direction due to i
        # the Sun's motion w.r.t the "kinematic" local standard of rest
        rvlsrk = s.sla_rvlsrk( rarad,dcrad)

        # rvgalc is velocity component in ra,dec direction due to
        #the rotation of the Galaxy.
        rvgalc = s.sla_rvgalc( rarad,dcrad)
        totalhelio = Rgeo + vcorhelio
        totalbary  = Rgeo + vcorbary
        totallsrk = totalhelio + rvlsrk
        totalgal  = totalbary  + rvlsrd + rvgalc

        return totallsrk


Instruction:
Create a Python 3 function to
blank DC bins in coarse channels.

Response:
def blank_dc(self, n_coarse_chan):
        """ Blank DC bins in coarse channels.

        Note: currently only works if entire file is read
        """

        if n_coarse_chan < 1:
            logger.warning('Coarse channel number < 1, unable to blank DC bin.')
            return None

        if not n_coarse_chan % int(n_coarse_chan) == 0:
            logger.warning('Selection does not contain an interger number of coarse channels, unable to blank DC bin.')
            return None

        n_coarse_chan = int(n_coarse_chan)

        n_chan = self.data.shape[-1]
        n_chan_per_coarse = int(n_chan / n_coarse_chan)

        mid_chan = int(n_chan_per_coarse / 2)

        for ii in range(n_coarse_chan):
            ss = ii*n_chan_per_coarse
            self.data[..., ss+mid_chan] = np.median(self.data[..., ss+mid_chan+5:ss+mid_chan+10])


Instruction:
Can you write a function in Python 3 where it
prints information about the current object.

Response:
def info(self):
        """ Print header information """

        for key, val in self.header.items():
            if key == b'src_raj':
                val = val.to_string(unit=u.hour, sep=':')
            if key == b'src_dej':
                val = val.to_string(unit=u.deg, sep=':')
            if key == b'tsamp':
                val *= u.second
            if key in ('foff', 'fch1'):
                val *= u.MHz
            if key == b'tstart':
                print("%16s : %32s" % ("tstart (ISOT)", Time(val, format='mjd').isot))
                key = "tstart (MJD)"
            print("%16s : %32s" % (key, val))

        print("\n%16s : %32s" % ("Num ints in file", self.n_ints_in_file))
        print("%16s : %32s" % ("Data shape", self.data.shape))
        print("%16s : %32s" % ("Start freq (MHz)", self.freqs[0]))
        print("%16s : %32s" % ("Stop freq (MHz)", self.freqs[-1]))


Instruction:
Can you generate the documentation for the following Python 3 function
def generate_freqs(self, f_start, f_stop):

        fch1 = self.header[b'fch1']
        foff = self.header[b'foff']

        #convert input frequencies into what their corresponding index would be
        i_start = int((f_start - fch1) / foff)
        i_stop  = int((f_stop - fch1)  / foff)

        #calculate closest true index value
        chan_start_idx = np.int(i_start)
        chan_stop_idx  = np.int(i_stop)

        #create freq array
        i_vals = np.arange(chan_stop_idx, chan_start_idx, 1)

        freqs = foff * i_vals + fch1

        return freqs

Response: Generate the frequency array for the given start and stop frequencies.


Instruction:
Implement a Python 3 function for
calculating the extent of the log entry.

Response:
def _calc_extent(self,plot_f=None,plot_t=None,MJD_time=False):
        """ Setup ploting edges.
        """

        plot_f_begin = plot_f[0]
        plot_f_end = plot_f[-1] + (plot_f[1]-plot_f[0])

        plot_t_begin = self.timestamps[0]
        plot_t_end  = self.timestamps[-1] + (self.timestamps[1] - self.timestamps[0])

        if MJD_time:
            extent=(plot_f_begin, plot_f_begin_end, plot_t_begin, plot_t_end)
        else:
            extent=(plot_f_begin, plot_f_end, 0.0,(plot_t_end-plot_t_begin)*24.*60.*60)

        return extent


Instruction:
Can you generate the documentation for the following Python 3 function
def plot_spectrum(self, t=0, f_start=None, f_stop=None, logged=False, if_id=0, c=None, **kwargs):
        if self.header[b'nbits'] <=2:
            logged = False
            t='all'
        ax = plt.gca()

        plot_f, plot_data = self.grab_data(f_start, f_stop, if_id)

        #Using accending frequency for all plots.
        if self.header[b'foff'] < 0:
            plot_data = plot_data[..., ::-1] # Reverse data
            plot_f = plot_f[::-1]

        if isinstance(t, int):
            print("extracting integration %i..." % t)
            plot_data = plot_data[t]
        elif t == b'all':
            print("averaging along time axis...")
            #Since the data has been squeezed, the axis for time goes away if only one bin, causing a bug with axis=1
            if len(plot_data.shape) > 1:
                plot_data = plot_data.mean(axis=0)
            else:
                plot_data = plot_data.mean()
        else:
            raise RuntimeError("Unknown integration %s" % t)

        # Rebin to max number of points
        dec_fac_x = 1
        if plot_data.shape[0] > MAX_PLT_POINTS:
            dec_fac_x = int(plot_data.shape[0] / MAX_PLT_POINTS)

        plot_data = rebin(plot_data, dec_fac_x, 1)
        plot_f    = rebin(plot_f, dec_fac_x, 1)

        if not c:
            kwargs['c'] = '#333333'

        if logged:
            plt.plot(plot_f, db(plot_data),label='Stokes I', **kwargs)
            plt.ylabel("Power [dB]")
        else:

            plt.plot(plot_f, plot_data,label='Stokes I', **kwargs)
            plt.ylabel("Power [counts]")
        plt.xlabel("Frequency [MHz]")
        plt.legend()

        try:
            plt.title(self.header[b'source_name'])
        except KeyError:
            plt.title(self.filename)

        plt.xlim(plot_f[0], plot_f[-1])

Response: Plot frequency spectrum of a given file.


Instruction:
Write a Python 3 script for
plotting the frequency spectrum of a given file.

Response:
def plot_spectrum_min_max(self, t=0, f_start=None, f_stop=None, logged=False, if_id=0, c=None, **kwargs):
        """ Plot frequency spectrum of a given file

        Args:
            logged (bool): Plot in linear (False) or dB units (True)
            if_id (int): IF identification (if multiple IF signals in file)
            c: color for line
            kwargs: keyword args to be passed to matplotlib plot()
        """
        ax = plt.gca()

        plot_f, plot_data = self.grab_data(f_start, f_stop, if_id)

        #Using accending frequency for all plots.
        if self.header[b'foff'] < 0:
            plot_data = plot_data[..., ::-1] # Reverse data
            plot_f = plot_f[::-1]

        fig_max = plot_data[0].max()
        fig_min = plot_data[0].min()

        print("averaging along time axis...")

        #Since the data has been squeezed, the axis for time goes away if only one bin, causing a bug with axis=1
        if len(plot_data.shape) > 1:
            plot_max = plot_data.max(axis=0)
            plot_min = plot_data.min(axis=0)
            plot_data = plot_data.mean(axis=0)
        else:
            plot_max = plot_data.max()
            plot_min = plot_data.min()
            plot_data = plot_data.mean()

        # Rebin to max number of points
        dec_fac_x = 1
        MAX_PLT_POINTS = 8*64  # Low resoluition to see the difference.
        if plot_data.shape[0] > MAX_PLT_POINTS:
            dec_fac_x = int(plot_data.shape[0] / MAX_PLT_POINTS)

        plot_data = rebin(plot_data, dec_fac_x, 1)
        plot_min = rebin(plot_min, dec_fac_x, 1)
        plot_max = rebin(plot_max, dec_fac_x, 1)
        plot_f    = rebin(plot_f, dec_fac_x, 1)

        if logged:
            plt.plot(plot_f, db(plot_data), "#333333", label='mean', **kwargs)
            plt.plot(plot_f, db(plot_max),  "#e74c3c", label='max', **kwargs)
            plt.plot(plot_f, db(plot_min),  '#3b5b92', label='min', **kwargs)
            plt.ylabel("Power [dB]")
        else:
            plt.plot(plot_f, plot_data,  "#333333", label='mean', **kwargs)
            plt.plot(plot_f, plot_max,   "#e74c3c", label='max', **kwargs)
            plt.plot(plot_f, plot_min,   '#3b5b92', label='min', **kwargs)
            plt.ylabel("Power [counts]")
        plt.xlabel("Frequency [MHz]")
        plt.legend()

        try:
            plt.title(self.header[b'source_name'])
        except KeyError:
            plt.title(self.filename)

        plt.xlim(plot_f[0], plot_f[-1])
        if logged:
            plt.ylim(db(fig_min),db(fig_max))


Instruction:
How would you implement a function in Python 3 that
plots the waterfall of the data in the specified frequency range.

Response:
def plot_waterfall(self, f_start=None, f_stop=None, if_id=0, logged=True, cb=True, MJD_time=False, **kwargs):
        """ Plot waterfall of data

        Args:
            f_start (float): start frequency, in MHz
            f_stop (float): stop frequency, in MHz
            logged (bool): Plot in linear (False) or dB units (True),
            cb (bool): for plotting the colorbar
            kwargs: keyword args to be passed to matplotlib imshow()
        """

        plot_f, plot_data = self.grab_data(f_start, f_stop, if_id)

        #Using accending frequency for all plots.
        if self.header[b'foff'] < 0:
            plot_data = plot_data[..., ::-1] # Reverse data
            plot_f = plot_f[::-1]

        if logged:
            plot_data = db(plot_data)

        # Make sure waterfall plot is under 4k*4k
        dec_fac_x, dec_fac_y = 1, 1
        if plot_data.shape[0] > MAX_IMSHOW_POINTS[0]:
            dec_fac_x = int(plot_data.shape[0] / MAX_IMSHOW_POINTS[0])

        if plot_data.shape[1] > MAX_IMSHOW_POINTS[1]:
            dec_fac_y =  int(plot_data.shape[1] /  MAX_IMSHOW_POINTS[1])

        plot_data = rebin(plot_data, dec_fac_x, dec_fac_y)

        try:
            plt.title(self.header[b'source_name'])
        except KeyError:
            plt.title(self.filename)

        extent = self._calc_extent(plot_f=plot_f,plot_t=self.timestamps,MJD_time=MJD_time)

        plt.imshow(plot_data,
            aspect='auto',
            origin='lower',
            rasterized=True,
            interpolation='nearest',
            extent=extent,
            cmap='viridis',
            **kwargs
        )
        if cb:
            plt.colorbar()
        plt.xlabel("Frequency [MHz]")
        if MJD_time:
            plt.ylabel("Time [MJD]")
        else:
            plt.ylabel("Time [s]")


Instruction:
Here you have a function in Python 3, explain what it does
def plot_time_series(self, f_start=None, f_stop=None, if_id=0, logged=True, orientation='h', MJD_time=False, **kwargs):

        ax = plt.gca()
        plot_f, plot_data = self.grab_data(f_start, f_stop, if_id)

        if logged and self.header[b'nbits'] >= 8:
            plot_data = db(plot_data)

        #Since the data has been squeezed, the axis for time goes away if only one bin, causing a bug with axis=1
        if len(plot_data.shape) > 1:
            plot_data = plot_data.mean(axis=1)
        else:
            plot_data = plot_data.mean()

        #Make proper time axis for plotting (but only for plotting!). Note that this makes the values inclusive.
        extent = self._calc_extent(plot_f=plot_f,plot_t=self.timestamps,MJD_time=MJD_time)
        plot_t = np.linspace(extent[2],extent[3],len(self.timestamps))

        if MJD_time:
            tlabel = "Time [MJD]"
        else:
            tlabel = "Time [s]"

        if logged:
            plabel = "Power [dB]"
        else:
            plabel = "Power [counts]"

        # Reverse oder if vertical orientation.
        if 'v' in orientation:
            plt.plot(plot_data, plot_t, **kwargs)
            plt.xlabel(plabel)

        else:
            plt.plot(plot_t, plot_data, **kwargs)
            plt.xlabel(tlabel)
            plt.ylabel(plabel)

        ax.autoscale(axis='both',tight=True)

Response: Plot the time series.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def plot_kurtosis(self, f_start=None, f_stop=None, if_id=0, **kwargs):
        ax = plt.gca()

        plot_f, plot_data = self.grab_data(f_start, f_stop, if_id)

        #Using accending frequency for all plots.
        if self.header[b'foff'] < 0:
            plot_data = plot_data[..., ::-1] # Reverse data
            plot_f = plot_f[::-1]

        try:
            plot_kurtosis = scipy.stats.kurtosis(plot_data, axis=0, nan_policy='omit')
        except:
            plot_kurtosis = plot_data*0.0

        plt.plot(plot_f, plot_kurtosis, **kwargs)
        plt.ylabel("Kurtosis")
        plt.xlabel("Frequency [MHz]")

        plt.xlim(plot_f[0], plot_f[-1])

Response: Plots the kurtosis of the current entry in the system.


Instruction:
Write a Python 3 script for
plotting all the data for a specific time series and kurtosis.

Response:
def plot_all(self, t=0, f_start=None, f_stop=None, logged=False, if_id=0, kurtosis=True, **kwargs):
        """ Plot waterfall of data as well as spectrum; also, placeholder to make even more complicated plots in the future.

        Args:
            f_start (float): start frequency, in MHz
            f_stop (float): stop frequency, in MHz
            logged (bool): Plot in linear (False) or dB units (True),
            t (int): integration number to plot (0 -> len(data))
            logged (bool): Plot in linear (False) or dB units (True)
            if_id (int): IF identification (if multiple IF signals in file)
            kwargs: keyword args to be passed to matplotlib plot() and imshow()
        """
        if self.header[b'nbits'] <=2:
            logged = False

        nullfmt = NullFormatter()  # no labels

        # definitions for the axes
        left, width = 0.35, 0.5
        bottom, height = 0.45, 0.5
        width2, height2 = 0.1125, 0.15
        bottom2, left2 = bottom - height2 - .025, left - width2 - .02
        bottom3, left3 = bottom2 - height2 - .025, 0.075

        rect_waterfall = [left, bottom, width, height]
        rect_colorbar = [left + width, bottom, .025, height]
        rect_spectrum = [left, bottom2, width, height2]
        rect_min_max = [left, bottom3, width, height2]
        rect_timeseries = [left + width, bottom, width2, height]
        rect_kurtosis = [left3, bottom3, 0.25, height2]
        rect_header = [left3 - .05, bottom, 0.2, height]

        # --------
        #         axColorbar = plt.axes(rect_colorbar)
        #         print 'Ploting Colorbar'
        #         print plot_data.max()
        #         print plot_data.min()
        #
        #         plot_colorbar = range(plot_data.min(),plot_data.max(),int((plot_data.max()-plot_data.min())/plot_data.shape[0]))
        #         plot_colorbar = np.array([[plot_colorbar],[plot_colorbar]])
        #
        #         plt.imshow(plot_colorbar,aspect='auto', rasterized=True, interpolation='nearest',)

        #         axColorbar.xaxis.set_major_formatter(nullfmt)
        #         axColorbar.yaxis.set_major_formatter(nullfmt)

        #         heatmap = axColorbar.pcolor(plot_data, edgecolors = 'none', picker=True)
        #         plt.colorbar(heatmap, cax = axColorbar)

        # --------
        axMinMax = plt.axes(rect_min_max)
        print('Plotting Min Max')
        self.plot_spectrum_min_max(logged=logged, f_start=f_start, f_stop=f_stop, t=t)
        plt.title('')
        axMinMax.yaxis.tick_right()
        axMinMax.yaxis.set_label_position("right")

        # --------
        axSpectrum = plt.axes(rect_spectrum,sharex=axMinMax)
        print('Plotting Spectrum')
        self.plot_spectrum(logged=logged, f_start=f_start, f_stop=f_stop, t=t)
        plt.title('')
        axSpectrum.yaxis.tick_right()
        axSpectrum.yaxis.set_label_position("right")
        plt.xlabel('')
#        axSpectrum.xaxis.set_major_formatter(nullfmt)
        plt.setp(axSpectrum.get_xticklabels(), visible=False)

        # --------
        axWaterfall = plt.axes(rect_waterfall,sharex=axMinMax)
        print('Plotting Waterfall')
        self.plot_waterfall(f_start=f_start, f_stop=f_stop, logged=logged, cb=False)
        plt.xlabel('')

        # no labels
#        axWaterfall.xaxis.set_major_formatter(nullfmt)
        plt.setp(axWaterfall.get_xticklabels(), visible=False)

        # --------
        axTimeseries = plt.axes(rect_timeseries)
        print('Plotting Timeseries')
        self.plot_time_series(f_start=f_start, f_stop=f_stop, orientation='v')
        axTimeseries.yaxis.set_major_formatter(nullfmt)
#        axTimeseries.xaxis.set_major_formatter(nullfmt)

        # --------
        # Could exclude since it takes much longer to run than the other plots.
        if kurtosis:
            axKurtosis = plt.axes(rect_kurtosis)
            print('Plotting Kurtosis')
            self.plot_kurtosis(f_start=f_start, f_stop=f_stop)

        # --------
        axHeader = plt.axes(rect_header)
        print('Plotting Header')
        # Generate nicer header
        telescopes = {0: 'Fake data',
                      1: 'Arecibo',
                      2: 'Ooty',
                      3: 'Nancay',
                      4: 'Parkes',
                      5: 'Jodrell',
                      6: 'GBT',
                      8: 'Effelsberg',
                      10: 'SRT',
                      64: 'MeerKAT',
                      65: 'KAT7'
                      }

        telescope = telescopes.get(self.header[b"telescope_id"], self.header[b"telescope_id"])

        plot_header = "%14s: %s\n" % ("TELESCOPE_ID", telescope)
        for key in (b'SRC_RAJ', b'SRC_DEJ', b'TSTART', b'NCHANS', b'NBEAMS', b'NIFS', b'NBITS'):
            try:
                plot_header += "%14s: %s\n" % (key, self.header[key.lower()])
            except KeyError:
                pass
        fch1 = "%6.6f MHz" % self.header[b'fch1']

        foff = (self.header[b'foff'] * 1e6 * u.Hz)
        if np.abs(foff) > 1e6 * u.Hz:
            foff = str(foff.to('MHz'))
        elif np.abs(foff) > 1e3 * u.Hz:
            foff = str(foff.to('kHz'))
        else:
            foff = str(foff.to('Hz'))

        plot_header += "%14s: %s\n" % ("FCH1", fch1)
        plot_header += "%14s: %s\n" % ("FOFF", foff)

        plt.text(0.05, .95, plot_header, ha='left', va='top', wrap=True)

        axHeader.set_facecolor('white')
        axHeader.xaxis.set_major_formatter(nullfmt)
        axHeader.yaxis.set_major_formatter(nullfmt)


Instruction:
Can you tell what is the following Python 3 function doing
def write_to_filterbank(self, filename_out):

        print("[Filterbank] Warning: Non-standard function to write in filterbank (.fil) format. Please use Waterfall.")

        n_bytes  = int(self.header[b'nbits'] / 8)
        with open(filename_out, "wb") as fileh:
            fileh.write(generate_sigproc_header(self))
            j = self.data
            if n_bytes == 4:
                np.float32(j.ravel()).tofile(fileh)
            elif n_bytes == 2:
                np.int16(j.ravel()).tofile(fileh)
            elif n_bytes == 1:
                np.int8(j.ravel()).tofile(fileh)

Response: Write data to blimpy file.


Instruction:
Implement a function in Python 3 to
write the filterbank data to an HDF5 file.

Response:
def write_to_hdf5(self, filename_out, *args, **kwargs):
        """ Write data to HDF5 file.

        Args:
            filename_out (str): Name of output file
        """

        print("[Filterbank] Warning: Non-standard function to write in HDF5 (.h5) format. Please use Waterfall.")

        if not HAS_HDF5:
            raise RuntimeError("h5py package required for HDF5 output.")

        with h5py.File(filename_out, 'w') as h5:

            dset = h5.create_dataset(b'data',
                              data=self.data,
                              compression='lzf')

            dset_mask = h5.create_dataset(b'mask',
                                     shape=self.data.shape,
                                     compression='lzf',
                                     dtype='uint8')

            dset.dims[0].label = b"frequency"
            dset.dims[1].label = b"feed_id"
            dset.dims[2].label = b"time"

            dset_mask.dims[0].label = b"frequency"
            dset_mask.dims[1].label = b"feed_id"
            dset_mask.dims[2].label = b"time"

            # Copy over header information as attributes
            for key, value in self.header.items():
                dset.attrs[key] = value


Instruction:
Explain what the following Python 3 code does
def calibrate_band_pass_N1(self):

        band_pass = np.median(self.data.squeeze(),axis=0)
        self.data = self.data/band_pass

Response: Calibration for the band pass of the log - level data.


Instruction:
Can you implement a function in Python 3 that
outputs stokes parameters I Q U V for a rawspec cross polarization filterbank file

Response:
def get_stokes(cross_dat, feedtype='l'):
    '''Output stokes parameters (I,Q,U,V) for a rawspec
    cross polarization filterbank file'''

    #Compute Stokes Parameters
    if feedtype=='l':
        #I = XX+YY
        I = cross_dat[:,0,:]+cross_dat[:,1,:]
        #Q = XX-YY
        Q = cross_dat[:,0,:]-cross_dat[:,1,:]
        #U = 2*Re(XY)
        U = 2*cross_dat[:,2,:]
        #V = -2*Im(XY)
        V = -2*cross_dat[:,3,:]

    elif feedtype=='c':
        #I = LL+RR
        I = cross_dat[:,0,:]+cross_dat[:,1,:]
        #Q = 2*Re(RL)
        Q = 2*cross_dat[:,2,:]
        #U = 2*Im(RL)
        U = -2*cross_dat[:,3,:]
        #V = RR-LL
        V = cross_dat[:,1,:]-cross_dat[:,0,:]
    else:
        raise ValueError('feedtype must be \'l\' (linear) or \'c\' (circular)')

    #Add middle dimension to match Filterbank format
    I = np.expand_dims(I,axis=1)
    Q = np.expand_dims(Q,axis=1)
    U = np.expand_dims(U,axis=1)
    V = np.expand_dims(V,axis=1)

    #Compute linear polarization
    #L=np.sqrt(np.square(Q)+np.square(U))

    return I,Q,U,V


Instruction:
How would you code a function in Python 3 to
convert a data array with length n_chans to an array of length n_coarse_chans by averaging over the coarse channels

Response:
def convert_to_coarse(data,chan_per_coarse):
    '''
    Converts a data array with length n_chans to an array of length n_coarse_chans
    by averaging over the coarse channels
    '''
    #find number of coarse channels and reshape array
    num_coarse = data.size/chan_per_coarse
    data_shaped = np.array(np.reshape(data,(num_coarse,chan_per_coarse)))

    #Return the average over each coarse channel
    return np.mean(data_shaped[:,2:-1],axis=1)


Instruction:
Here you have a function in Python 3, explain what it does
def phase_offsets(Idat,Qdat,Udat,Vdat,tsamp,chan_per_coarse,feedtype='l',**kwargs):
    '''
    Calculates phase difference between X and Y feeds given U and V (U and Q for circular basis)
    data from a noise diode measurement on the target
    '''
    #Fold noise diode data and calculate ON OFF diferences for U and V
    if feedtype=='l':
        U_OFF,U_ON = foldcal(Udat,tsamp,**kwargs)
        V_OFF,V_ON = foldcal(Vdat,tsamp,**kwargs)
        Udiff = U_ON-U_OFF
        Vdiff = V_ON-V_OFF
        poffset = np.arctan2(-1*Vdiff,Udiff)

    if feedtype=='c':
        U_OFF,U_ON = foldcal(Udat,tsamp,**kwargs)
        Q_OFF,Q_ON = foldcal(Qdat,tsamp,**kwargs)
        Udiff = U_ON-U_OFF
        Qdiff = Q_ON-Q_OFF
        poffset = np.arctan2(Udiff,Qdiff)

    coarse_p =  convert_to_coarse(poffset,chan_per_coarse)

    #Correct for problems created by discontinuity in arctan
    #Find whether phase offsets have increasing or decreasing slope
    y = coarse_p[:6]
    x = np.arange(y.size)
    m = np.polyfit(x,y,1)[0]

    for i in range(coarse_p.size-3):
        if (m>0 and coarse_p[i+1]<coarse_p[i]) or (m<0 and coarse_p[i+1]>coarse_p[i]):
            coarse_p[i+1] = 2*coarse_p[i+2]-coarse_p[i+3]    #Move problem point near the next

    return coarse_p

Response: Calculates the phase difference between X and Y feeds given U and V and a frequency series of frequency intervals.