Instruction:
How would you explain what the following Python 3 function does
def arxiv_categories(self, key, value):
    def _is_arxiv(category):
        return category in valid_arxiv_categories()

    def _is_inspire(category):
        schema = load_schema('elements/inspire_field')
        valid_inspire_categories = schema['properties']['term']['enum']

        return category in valid_inspire_categories

    def _normalize(a_value):
        for category in valid_arxiv_categories():
            if a_value.lower() == category.lower():
                return normalize_arxiv_category(category)

        schema = load_schema('elements/inspire_field')
        valid_inspire_categories = schema['properties']['term']['enum']

        for category in valid_inspire_categories:
            if a_value.lower() == category.lower():
                return category

        field_codes_to_inspire_categories = {
            'a': 'Astrophysics',
            'b': 'Accelerators',
            'c': 'Computing',
            'e': 'Experiment-HEP',
            'g': 'Gravitation and Cosmology',
            'i': 'Instrumentation',
            'l': 'Lattice',
            'm': 'Math and Math Physics',
            'n': 'Theory-Nucl',
            'o': 'Other',
            'p': 'Phenomenology-HEP',
            'q': 'General Physics',
            't': 'Theory-HEP',
            'x': 'Experiment-Nucl',
        }

        return field_codes_to_inspire_categories.get(a_value.lower())

    arxiv_categories = self.get('arxiv_categories', [])
    inspire_categories = self.get('inspire_categories', [])

    for value in force_list(value):
        for a_value in force_list(value.get('a')):
            normalized_a_value = _normalize(a_value)

            if _is_arxiv(normalized_a_value):
                arxiv_categories.append(normalized_a_value)
            elif _is_inspire(normalized_a_value):
                inspire_categories.append({'term': normalized_a_value})

    self['inspire_categories'] = inspire_categories
    return arxiv_categories

Response: Populate the arxiv_categories key.


Instruction:
Can you create a Python 3 function that
populates the 100__d MARC field.

Response:
def birth_and_death_date2marc(self, key, value):
    """Populate the ``100__d`` MARC field, which includes the birth and the death date.

    By not using the decorator ```for_each_value```, the values of the fields
    ```birth_date``` and ```death_date``` are both added to ```values``` as a list.
    """
    name_field = self.get('100', {})

    if 'd' in name_field:
        if int(name_field['d'].split('-')[0]) > int(value.split('-')[0]):
            dates_field = ' - '.join([value, name_field['d']])
        else:
            dates_field = ' - '.join([name_field['d'], value])
    else:
        dates_field = value

    name_field['d'] = dates_field

    return name_field


Instruction:
Write a Python 3 script for
populating the url key.

Response:
def urls(self, key, value):
    """Populate the ``url`` key.

    Also populates the ``ids`` key through side effects.
    """
    description = force_single_element(value.get('y'))
    url = value.get('u')

    linkedin_match = LINKEDIN_URL.match(url)
    twitter_match = TWITTER_URL.match(url)
    wikipedia_match = WIKIPEDIA_URL.match(url)
    if linkedin_match:
        self.setdefault('ids', []).append(
            {
                'schema': 'LINKEDIN',
                'value': unquote_url(linkedin_match.group('page')),
            }
        )
    elif twitter_match:
        self.setdefault('ids', []).append(
            {
                'schema': 'TWITTER',
                'value': twitter_match.group('handle'),
            }
        )
    elif wikipedia_match:
        lang = wikipedia_match.group('lang')
        page = unquote_url(wikipedia_match.group('page'))
        if lang != 'en':
            page = ':'.join([lang, page])
        self.setdefault('ids', []).append(
            {
                'schema': 'WIKIPEDIA',
                'value': page,
            }
        )
    else:
        return {
            'description': description,
            'value': url,
        }


Instruction:
Can you tell what is the following Python 3 function doing
def new_record(self, key, value):
    new_record = self.get('new_record', {})
    ids = self.get('ids', [])

    for value in force_list(value):
        for id_ in force_list(value.get('a')):
            ids.append({
                'schema': 'SPIRES',
                'value': id_,
            })

        new_recid = force_single_element(value.get('d', ''))
        if new_recid:
            new_record = get_record_ref(new_recid, 'authors')

    self['ids'] = ids
    return new_record

Response: Populate the new_record key.


Instruction:
Create a Python 3 function for
populating the deleted key.

Response:
def deleted(self, key, value):
    """Populate the ``deleted`` key.

    Also populates the ``stub`` key through side effects.
    """
    def _is_deleted(value):
        return force_single_element(value.get('c', '')).upper() == 'DELETED'

    def _is_stub(value):
        return not (force_single_element(value.get('a', '')).upper() == 'USEFUL')

    deleted = self.get('deleted')
    stub = self.get('stub')

    for value in force_list(value):
        deleted = not deleted and _is_deleted(value)
        stub = not stub and _is_stub(value)

    self['stub'] = stub
    return deleted


Instruction:
Can you tell what is the following Python 3 function doing
def authors2marc(self, key, value):
    value = force_list(value)

    def _get_ids(value):
        ids = {
            'i': [],
            'j': [],
        }
        if value.get('ids'):
            for _id in value.get('ids'):
                if _id.get('schema') == 'INSPIRE ID':
                    ids['i'].append(_id.get('value'))
                elif _id.get('schema') == 'ORCID':
                    ids['j'].append('ORCID:' + _id.get('value'))
                elif _id.get('schema') == 'JACOW':
                    ids['j'].append(_id.get('value'))
                elif _id.get('schema') == 'CERN':
                    ids['j'].append('CCID-' + _id.get('value')[5:])
        return ids

    def _get_affiliations(value):
        return [
            aff.get('value') for aff in value.get('affiliations', [])
        ]

    def _get_affiliations_identifiers(value):
        return [
            u'{}:{}'.format(aff.get('schema'), aff.get('value')) for aff in value.get('affiliations_identifiers', [])
        ]

    def _get_inspire_roles(value):
        values = force_list(value.get('inspire_roles'))
        return ['ed.' for role in values if role == 'editor']

    def _get_raw_affiliations(value):
        return [
            aff.get('value') for aff in value.get('raw_affiliations', [])
        ]

    def get_value_100_700(value):
        ids = _get_ids(value)
        return {
            'a': value.get('full_name'),
            'e': _get_inspire_roles(value),
            'q': value.get('alternative_names'),
            'i': ids.get('i'),
            'j': ids.get('j'),
            'm': value.get('emails'),
            't': _get_affiliations_identifiers(value),
            'u': _get_affiliations(value),
            'v': _get_raw_affiliations(value),
        }

    def get_value_701(value):
        ids = _get_ids(value)
        return {
            'a': value.get('full_name'),
            'q': value.get('alternative_names'),
            'i': ids.get('i'),
            'j': ids.get('j'),
            'u': _get_affiliations(value),
            'v': _get_raw_affiliations(value),
        }

    if len(value) > 1:
        self["700"] = []
        self["701"] = []

    for author in value[1:]:
        is_supervisor = 'supervisor' in author.get('inspire_roles', [])
        if is_supervisor:
            self["701"].append(get_value_701(author))
        else:
            self["700"].append(get_value_100_700(author))
    return get_value_100_700(value[0])

Response: Populate the 100 MARC field.


Instruction:
Explain what the following Python 3 code does
def isbns(self, key, value):
    def _get_medium(value):
        def _normalize(medium):
            schema = load_schema('hep')
            valid_media = schema['properties']['isbns']['items']['properties']['medium']['enum']

            medium = medium.lower().replace('-', '').replace(' ', '')
            if medium in valid_media:
                return medium
            elif medium == 'ebook':
                return 'online'
            elif medium == 'paperback':
                return 'softcover'

            return ''

        medium = force_single_element(value.get('b', ''))
        normalized_medium = _normalize(medium)

        return normalized_medium

    def _get_isbn(value):
        a_value = force_single_element(value.get('a', ''))
        normalized_a_value = a_value.replace('.', '')
        if normalized_a_value:
            return normalize_isbn(normalized_a_value)

    return {
        'medium': _get_medium(value),
        'value': _get_isbn(value),
    }

Response: Populate the isbns key.


Instruction:
Can you tell what is the following Python 3 function doing
def dois(self, key, value):
    def _get_first_non_curator_source(sources):
        sources_without_curator = [el for el in sources if el.upper() != 'CURATOR']
        return force_single_element(sources_without_curator)

    def _get_material(value):
        MATERIAL_MAP = {
            'ebook': 'publication',
        }

        q_value = force_single_element(value.get('q', ''))
        normalized_q_value = q_value.lower()

        return MATERIAL_MAP.get(normalized_q_value, normalized_q_value)

    def _is_doi(id_, type_):
        return (not type_ or type_.upper() == 'DOI') and is_doi(id_)

    def _is_handle(id_, type_):
        return (not type_ or type_.upper() == 'HDL') and is_handle(id_)

    dois = self.get('dois', [])
    persistent_identifiers = self.get('persistent_identifiers', [])

    values = force_list(value)
    for value in values:
        id_ = force_single_element(value.get('a', ''))
        material = _get_material(value)
        schema = force_single_element(value.get('2', ''))

        sources = force_list(value.get('9'))
        source = _get_first_non_curator_source(sources)

        if _is_doi(id_, schema):
            dois.append({
                'material': material,
                'source': source,
                'value': normalize_doi(id_),
            })
        else:
            schema = 'HDL' if _is_handle(id_, schema) else schema
            persistent_identifiers.append({
                'material': material,
                'schema': schema,
                'source': source,
                'value': id_,
            })

    self['persistent_identifiers'] = persistent_identifiers
    return dois

Response: Populate the dois key.


Instruction:
Create a Python 3 function for
populating the 0247 MARC field.

Response:
def dois2marc(self, key, value):
    """Populate the ``0247`` MARC field."""
    return {
        '2': 'DOI',
        '9': value.get('source'),
        'a': value.get('value'),
        'q': value.get('material'),
    }


Instruction:
Write a Python 3 function for
populating the 0247 MARC field.

Response:
def persistent_identifiers2marc(self, key, value):
    """Populate the ``0247`` MARC field."""
    return {
        '2': value.get('schema'),
        '9': value.get('source'),
        'a': value.get('value'),
        'q': value.get('material'),
    }


Instruction:
Here you have a function in Python 3, explain what it does
def texkeys(self, key, value):
    def _is_oai(id_, schema):
        return id_.startswith('oai:')

    def _is_desy(id_, schema):
        return id_ and schema in ('DESY',)

    def _is_texkey(id_, schema):
        return id_ and schema in ('INSPIRETeX', 'SPIRESTeX')

    texkeys = self.get('texkeys', [])
    external_system_identifiers = self.get('external_system_identifiers', [])
    _desy_bookkeeping = self.get('_desy_bookkeeping', [])

    values = force_list(value)
    for value in values:
        ids = force_list(value.get('a', ''))
        other_ids = force_list(value.get('z', ''))
        schema = force_single_element(value.get('9', ''))

        for id_ in ids:
            id_ = id_.strip()
            if not id_:
                continue

            if _is_texkey(id_, schema):
                texkeys.insert(0, id_)
            elif _is_oai(id_, schema):
                continue  # XXX: ignored.
            elif _is_desy(id_, schema):
                _desy_bookkeeping.append({'identifier': id_})
            else:
                external_system_identifiers.insert(0, {
                    'schema': schema,
                    'value': id_,
                })

        for id_ in other_ids:
            id_ = id_.strip()
            if not id_:
                continue

            if _is_texkey(id_, schema):
                texkeys.append(id_)
            elif _is_oai(id_, schema):
                continue  # XXX: ignored.
            elif _is_desy(id_, schema):
                _desy_bookkeeping.append({'identifier': id_})
            else:
                external_system_identifiers.append({
                    'schema': schema,
                    'value': id_,
                })

    self['external_system_identifiers'] = external_system_identifiers
    self['_desy_bookkeeping'] = _desy_bookkeeping
    return texkeys

Response: Populate the texkeys key.


Instruction:
Explain what the following Python 3 code does
def texkeys2marc(self, key, value):
    result = []

    values = force_list(value)
    if values:
        value = values[0]
        result.append({
            '9': 'INSPIRETeX',
            'a': value,
        })

        for value in values[1:]:
            result.append({
                '9': 'INSPIRETeX',
                'z': value,
            })

    return result

Response: Populate the 3035 MARC field.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def external_system_identifiers2marc(self, key, value):
    def _is_scheme_cernkey(id_, schema):
        return schema == 'CERNKEY'

    def _is_scheme_spires(id_, schema):
        return schema == 'SPIRES'

    result_035 = self.get('035', [])
    id_dict = self.get('id_dict', defaultdict(list))
    result_970 = self.get('970', [])

    values = force_list(value)
    for value in values:
        id_ = value.get('value')
        schema = value.get('schema')

        if _is_scheme_spires(id_, schema):
            result_970.append({
                'a': id_,
            })
        elif _is_scheme_cernkey(id_, schema):
            result_035.append({
                '9': 'CERNKEY',
                'z': id_,
            })
        else:
            id_dict[schema].append(id_)

    self['970'] = result_970
    self['id_dict'] = id_dict
    return result_035

Response: Populate the 970 MARC field based on external system identifiers.


Instruction:
Given the following Python 3 function, write the documentation
def arxiv_eprints(self, key, value):
    def _get_clean_arxiv_eprint(id_):
        return id_.split(':')[-1]

    def _is_arxiv_eprint(id_, source):
        return source.lower() == 'arxiv'

    def _is_hidden_report_number(other_id, source):
        return other_id

    def _get_clean_source(source):
        if source == 'arXiv:reportnumber':
            return 'arXiv'
        return source

    arxiv_eprints = self.get('arxiv_eprints', [])
    report_numbers = self.get('report_numbers', [])

    values = force_list(value)
    for value in values:
        id_ = force_single_element(value.get('a', ''))
        other_id = force_single_element(value.get('z', ''))
        categories = [normalize_arxiv_category(category) for category
                      in force_list(value.get('c'))]
        source = force_single_element(value.get('9', ''))

        if _is_arxiv_eprint(id_, source):
            arxiv_eprints.append({
                'categories': categories,
                'value': _get_clean_arxiv_eprint(id_),
            })
        elif _is_hidden_report_number(other_id, source):
            report_numbers.append({
                'hidden': True,
                'source': _get_clean_source(source),
                'value': other_id,
            })
        else:
            report_numbers.append({
                'source': _get_clean_source(source),
                'value': id_,
            })

    self['report_numbers'] = report_numbers
    return arxiv_eprints

Response: Populate the arxiv_eprints key.


Instruction:
Can you implement a function in Python 3 that
populates the 3037 MARC field.

Response:
def arxiv_eprints2marc(self, key, values):
    """Populate the ``037`` MARC field.

    Also populates the ``035`` and the ``65017`` MARC fields through side effects.
    """
    result_037 = self.get('037', [])
    result_035 = self.get('035', [])
    result_65017 = self.get('65017', [])

    for value in values:
        arxiv_id = value.get('value')
        arxiv_id = 'arXiv:' + arxiv_id if is_arxiv_post_2007(arxiv_id) else arxiv_id
        result_037.append({
            '9': 'arXiv',
            'a': arxiv_id,
            'c': force_single_element(value.get('categories')),
        })

        result_035.append({
            '9': 'arXiv',
            'a': 'oai:arXiv.org:' + value.get('value'),
        })

        categories = force_list(value.get('categories'))
        for category in categories:
            result_65017.append({
                '2': 'arXiv',
                'a': category,
            })

    self['65017'] = result_65017
    self['035'] = result_035
    return result_037


Instruction:
Implement a Python 3 function for
populating the 3037 MARC field.

Response:
def report_numbers2marc(self, key, value):
    """Populate the ``037`` MARC field."""
    def _get_mangled_source(source):
        if source == 'arXiv':
            return 'arXiv:reportnumber'
        return source

    source = _get_mangled_source(value.get('source'))

    if value.get('hidden'):
        return {
            '9': source,
            'z': value.get('value'),
        }

    return {
        '9': source,
        'a': value.get('value'),
    }


Instruction:
Make a summary of the following Python 3 code
def languages(self, key, value):
    languages = self.get('languages', [])

    values = force_list(value.get('a'))
    for value in values:
        for language in RE_LANGUAGE.split(value):
            try:
                name = language.strip().capitalize()
                languages.append(pycountry.languages.get(name=name).alpha_2)
            except KeyError:
                pass

    return languages

Response: Populate the languages key.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def languages2marc(self, key, value):
    return {'a': pycountry.languages.get(alpha_2=value).name.lower()}

Response: Populate the 041 MARC field.


Instruction:
Explain what the following Python 3 code does
def record_affiliations(self, key, value):
    record = get_record_ref(value.get('z'), 'institutions')

    return {
        'curated_relation': record is not None,
        'record': record,
        'value': value.get('a'),
    }

Response: Populate the record_affiliations key.


Instruction:
Can you write a function in Python 3 where it
populates the document_type key.

Response:
def document_type(self, key, value):
    """Populate the ``document_type`` key.

    Also populates the ``_collections``, ``citeable``, ``core``, ``deleted``,
    ``refereed``, ``publication_type``, and ``withdrawn`` keys through side
    effects.
    """
    schema = load_schema('hep')
    publication_type_schema = schema['properties']['publication_type']
    valid_publication_types = publication_type_schema['items']['enum']

    document_type = self.get('document_type', [])
    publication_type = self.get('publication_type', [])

    a_values = force_list(value.get('a'))
    for a_value in a_values:
        normalized_a_value = a_value.strip().lower()

        if normalized_a_value == 'arxiv':
            continue  # XXX: ignored.
        elif normalized_a_value == 'citeable':
            self['citeable'] = True
        elif normalized_a_value == 'core':
            self['core'] = True
        elif normalized_a_value == 'noncore':
            self['core'] = False
        elif normalized_a_value == 'published':
            self['refereed'] = True
        elif normalized_a_value == 'withdrawn':
            self['withdrawn'] = True
        elif normalized_a_value == 'deleted':
            self['deleted'] = True
        elif normalized_a_value in COLLECTIONS_MAP:
            self.setdefault('_collections', []).append(COLLECTIONS_MAP[normalized_a_value])
        elif normalized_a_value in DOCUMENT_TYPE_MAP:
            document_type.append(DOCUMENT_TYPE_MAP[normalized_a_value])
        elif normalized_a_value in valid_publication_types:
            publication_type.append(normalized_a_value)

    c_value = force_single_element(value.get('c', ''))
    normalized_c_value = c_value.strip().lower()

    if normalized_c_value == 'deleted':
        self['deleted'] = True

    self['publication_type'] = publication_type
    return document_type


Instruction:
Explain what the following Python 3 code does
def document_type2marc(self, key, value):
    if value in DOCUMENT_TYPE_REVERSE_MAP and DOCUMENT_TYPE_REVERSE_MAP[value]:
        return {'a': DOCUMENT_TYPE_REVERSE_MAP[value]}

Response: Populate the 980 MARC field.


Instruction:
Can you create a Python 3 function that
populates the references key.

Response:
def references(self, key, value):
    """Populate the ``references`` key."""
    def _has_curator_flag(value):
        normalized_nine_values = [el.upper() for el in force_list(value.get('9'))]
        return 'CURATOR' in normalized_nine_values

    def _is_curated(value):
        return force_single_element(value.get('z')) == '1' and _has_curator_flag(value)

    def _set_record(el):
        recid = maybe_int(el)
        record = get_record_ref(recid, 'literature')
        rb.set_record(record)

    rb = ReferenceBuilder()
    mapping = [
        ('0', _set_record),
        ('a', rb.add_uid),
        ('b', rb.add_uid),
        ('c', rb.add_collaboration),
        ('e', partial(rb.add_author, role='ed.')),
        ('h', rb.add_refextract_authors_str),
        ('i', rb.add_uid),
        ('k', rb.set_texkey),
        ('m', rb.add_misc),
        ('o', rb.set_label),
        ('p', rb.set_publisher),
        ('q', rb.add_parent_title),
        ('r', rb.add_report_number),
        ('s', rb.set_pubnote),
        ('t', rb.add_title),
        ('x', rb.add_raw_reference),
        ('y', rb.set_year),
    ]

    for field, method in mapping:
        for el in force_list(value.get(field)):
            if el:
                method(el)

    for el in dedupe_list(force_list(value.get('u'))):
        if el:
            rb.add_url(el)

    if _is_curated(value):
        rb.curate()

    if _has_curator_flag(value):
        rb.obj['legacy_curated'] = True

    return rb.obj


Instruction:
Create a Python 3 function for
populating the 999C5 MARC field.

Response:
def references2marc(self, key, value):
    """Populate the ``999C5`` MARC field."""
    reference = value.get('reference', {})

    pids = force_list(reference.get('persistent_identifiers'))
    a_values = ['doi:' + el for el in force_list(reference.get('dois'))]
    a_values.extend(['hdl:' + el['value'] for el in pids if el.get('schema') == 'HDL'])
    a_values.extend(['urn:' + el['value'] for el in pids if el.get('schema') == 'URN'])

    external_ids = force_list(reference.get('external_system_identifiers'))
    u_values = force_list(get_value(reference, 'urls.value'))
    u_values.extend(CDS_RECORD_FORMAT.format(el['value']) for el in external_ids if el.get('schema') == 'CDS')
    u_values.extend(ADS_RECORD_FORMAT.format(el['value']) for el in external_ids if el.get('schema') == 'ADS')

    authors = force_list(reference.get('authors'))
    e_values = [el['full_name'] for el in authors if el.get('inspire_role') == 'editor']
    h_values = [el['full_name'] for el in authors if el.get('inspire_role') != 'editor']

    r_values = force_list(reference.get('report_numbers'))
    if reference.get('arxiv_eprint'):
        arxiv_eprint = reference['arxiv_eprint']
        r_values.append('arXiv:' + arxiv_eprint if is_arxiv_post_2007(arxiv_eprint) else arxiv_eprint)

    if reference.get('publication_info'):
        reference['publication_info'] = convert_new_publication_info_to_old([reference['publication_info']])[0]
    journal_title = get_value(reference, 'publication_info.journal_title')
    journal_volume = get_value(reference, 'publication_info.journal_volume')
    page_start = get_value(reference, 'publication_info.page_start')
    page_end = get_value(reference, 'publication_info.page_end')
    artid = get_value(reference, 'publication_info.artid')
    s_value = build_pubnote(journal_title, journal_volume, page_start, page_end, artid)

    m_value = ' / '.join(force_list(reference.get('misc')))

    return {
        '0': get_recid_from_ref(value.get('record')),
        '9': 'CURATOR' if value.get('legacy_curated') else None,
        'a': a_values,
        'b': get_value(reference, 'publication_info.cnum'),
        'c': reference.get('collaborations'),
        'e': e_values,
        'h': h_values,
        'i': reference.get('isbn'),
        'k': reference.get('texkey'),
        'm': m_value,
        'o': reference.get('label'),
        'p': get_value(reference, 'imprint.publisher'),
        'q': get_value(reference, 'publication_info.parent_title'),
        'r': r_values,
        's': s_value,
        't': get_value(reference, 'title.title'),
        'u': u_values,
        'x': get_value(value, 'raw_refs.value'),
        'y': get_value(reference, 'publication_info.year'),
        'z': 1 if value.get('curated_relation') else 0,
    }


Instruction:
Make a summary of the following Python 3 code
def documents(self, key, value):
    def _is_hidden(value):
        return 'HIDDEN' in [val.upper() for val in value] or None

    def _is_figure(value):
        figures_extensions = ['.png']
        return value.get('f') in figures_extensions

    def _is_fulltext(value):
        return value.get('d', '').lower() == 'fulltext' or None

    def _get_index_and_caption(value):
        match = re.compile(r'(^\d{5})?\s*(.*)').match(value)
        if match:
            return match.group(1), match.group(2)

    def _get_key(value):
        fname = value.get('n', 'document')
        extension = value.get('f', '')

        if fname.endswith(extension):
            return fname
        return fname + extension

    def _get_source(value):
        source = value.get('t', '')
        if source in ('INSPIRE-PUBLIC', 'Main'):
            source = None
        elif source.lower() == 'arxiv':
            return 'arxiv'

        return source

    figures = self.get('figures', [])
    is_context = value.get('f', '').endswith('context')

    if is_context:
        return

    if _is_figure(value):
        index, caption = _get_index_and_caption(value.get('d', ''))
        figures.append({
            'key': _get_key(value),
            'caption': caption,
            'url': afs_url(value.get('a')),
            'order': index,
            'source': 'arxiv',  # XXX: we don't have any other figures on legacy
        })
        self['figures'] = figures
    else:
        return {
            'description': value.get('d') if not _is_fulltext(value) else None,
            'key': _get_key(value),
            'fulltext': _is_fulltext(value),
            'hidden': _is_hidden(force_list(value.get('o'))),
            'url': afs_url(value.get('a')),
            'source': _get_source(value),
        }

Response: Populate the documents key.


Instruction:
Can you tell what is the following Python 3 function doing
def marcxml2record(marcxml):
    marcjson = create_record(marcxml, keep_singletons=False)
    collections = _get_collections(marcjson)

    if 'conferences' in collections:
        return conferences.do(marcjson)
    elif 'data' in collections:
        return data.do(marcjson)
    elif 'experiment' in collections:
        return experiments.do(marcjson)
    elif 'hepnames' in collections:
        return hepnames.do(marcjson)
    elif 'institution' in collections:
        return institutions.do(marcjson)
    elif 'job' in collections or 'jobhidden' in collections:
        return jobs.do(marcjson)
    elif 'journals' in collections or 'journalsnew' in collections:
        return journals.do(marcjson)
    return hep.do(marcjson)

Response: Convert a MARCXML string to a JSON record.


Instruction:
Write a Python 3 script for
converting a JSON record to a MARCXML string.

Response:
def record2marcxml(record):
    """Convert a JSON record to a MARCXML string.

    Deduces which set of rules to use by parsing the ``$schema`` key, as
    it unequivocally determines which kind of record we have.

    Args:
        record(dict): a JSON record.

    Returns:
        str: a MARCXML string converted from the record.

    """
    schema_name = _get_schema_name(record)

    if schema_name == 'hep':
        marcjson = hep2marc.do(record)
    elif schema_name == 'authors':
        marcjson = hepnames2marc.do(record)
    else:
        raise NotImplementedError(u'JSON -> MARC rules missing for "{}"'.format(schema_name))

    record = RECORD()

    for key, values in sorted(iteritems(marcjson)):
        tag, ind1, ind2 = _parse_key(key)
        if _is_controlfield(tag, ind1, ind2):
            value = force_single_element(values)
            if not isinstance(value, text_type):
                value = text_type(value)
            record.append(CONTROLFIELD(_strip_invalid_chars_for_xml(value), {'tag': tag}))
        else:
            for value in force_list(values):
                datafield = DATAFIELD({'tag': tag, 'ind1': ind1, 'ind2': ind2})
                for code, els in sorted(iteritems(value)):
                    for el in force_list(els):
                        if not isinstance(el, text_type):
                            el = text_type(el)
                        datafield.append(SUBFIELD(_strip_invalid_chars_for_xml(el), {'code': code}))
                record.append(datafield)

    return tostring(record, encoding='utf8', pretty_print=True)


Instruction:
How would you code a function in Python 3 to
populate the number_of_pages key.

Response:
def number_of_pages(self, key, value):
    """Populate the ``number_of_pages`` key."""
    result = maybe_int(force_single_element(value.get('a', '')))
    if result and result > 0:
        return result


Instruction:
Given the following Python 3 function, write the documentation
def secondary_report_numbers(self, key, value):
    preliminary_results_prefixes = ['ATLAS-CONF-', 'CMS-PAS-', 'CMS-DP-', 'LHCB-CONF-']
    note_prefixes = ['ALICE-INT-', 'ATL-', 'ATLAS-CONF-', 'CMS-DP-', 'CMS-PAS-', 'LHCB-CONF-', 'LHCB-PUB-']

    result_037 = self.get('037__', [])
    result_500 = self.get('500__', [])
    result_595 = self.get('595__', [])
    result_980 = self.get('980__', [])

    report = force_single_element(value.get('a', ''))
    hidden_report = force_single_element(value.get('9') or value.get('z', ''))
    source = 'CDS' if not is_arxiv(report) else 'arXiv'

    if any(report.upper().startswith(prefix) for prefix in note_prefixes):
        result_980.append({'a': 'NOTE'})

    if any(report.upper().startswith(prefix) for prefix in preliminary_results_prefixes):
        result_500.append({'9': 'CDS', 'a': 'Preliminary results'})

    is_barcode = hidden_report.startswith('P0') or hidden_report.startswith('CM-P0')
    if not report.startswith('SIS-') and not is_barcode:
        result_037.append({
            '9': source,
            'a': report,
            'c': value.get('c'),
            'z': hidden_report if source == 'CDS' else None,
        })

    self['500__'] = result_500
    self['595__'] = result_595
    self['980__'] = result_980
    return result_037

Response: Populate the 216 MARC field.


Instruction:
Can you create a Python 3 function that
populates the 701 MARC field.

Response:
def nonfirst_authors(self, key, value):
    """Populate ``700`` MARC field.

    Also populates the ``701`` MARC field through side-effects.
    """
    field_700 = self.get('700__', [])
    field_701 = self.get('701__', [])

    is_supervisor = any(el.lower().startswith('dir') for el in force_list(value.get('e', '')))
    if is_supervisor:
        field_701.append(_converted_author(value))
    else:
        field_700.append(_converted_author(value))

    self['701__'] = field_701
    return field_700


Instruction:
How would you explain what the following Python 3 function does
def urls(self, key, value):
    def _is_preprint(value):
        return value.get('y', '').lower() == 'preprint'

    def _is_fulltext(value):
        return value['u'].endswith('.pdf') and value['u'].startswith('http://cds.cern.ch')

    def _is_local_copy(value):
        return 'local copy' in value.get('y', '')

    def _is_ignored_domain(value):
        ignored_domains = ['http://cdsweb.cern.ch', 'http://cms.cern.ch',
                           'http://cmsdoc.cern.ch', 'http://documents.cern.ch',
                           'http://preprints.cern.ch', 'http://cds.cern.ch',
                           'http://arxiv.org']
        return any(value['u'].startswith(domain) for domain in ignored_domains)

    field_8564 = self.get('8564_', [])
    field_FFT = self.get('FFT__', [])

    if 'u' not in value:
        return field_8564

    url = escape_url(value['u'])

    if _is_fulltext(value) and not _is_preprint(value):
        if _is_local_copy(value):
            description = value.get('y', '').replace('local copy', 'on CERN Document Server')
            field_8564.append({
                'u': url,
                'y': description,
            })
        else:
            _, file_name = os.path.split(urllib.parse.urlparse(value['u']).path)
            _, extension = os.path.splitext(file_name)
            field_FFT.append({
                't': 'CDS',
                'a': url,
                'd': value.get('y', ''),
                'n': file_name,
                'f': extension,
            })
    elif not _is_ignored_domain(value):
        field_8564.append({
            'u': url,
            'y': value.get('y'),
        })

    self['FFT__'] = field_FFT
    return field_8564

Response: Populate the 8564 MARC field.


Instruction:
Create a Python 3 function to
populate the titles key.

Response:
def titles(self, key, value):
    """Populate the ``titles`` key."""
    if not key.startswith('245'):
        return {
            'source': value.get('9'),
            'subtitle': value.get('b'),
            'title': value.get('a'),
        }

    self.setdefault('titles', []).insert(0, {
        'source': value.get('9'),
        'subtitle': value.get('b'),
        'title': value.get('a'),
    })


Instruction:
Given the following Python 3 function, write the documentation
def title_translations(self, key, value):
    return {
        'language': langdetect.detect(value.get('a')),
        'source': value.get('9'),
        'subtitle': value.get('b'),
        'title': value.get('a'),
    }

Response: Populate the title_translations key.


Instruction:
Can you implement a function in Python 3 that
populates the 246 MARC field from a list of titles.

Response:
def titles2marc(self, key, values):
    """Populate the ``246`` MARC field.

    Also populates the ``245`` MARC field through side effects.
    """
    first, rest = values[0], values[1:]

    self.setdefault('245', []).append({
        'a': first.get('title'),
        'b': first.get('subtitle'),
        '9': first.get('source'),
    })

    return [
        {
            'a': value.get('title'),
            'b': value.get('subtitle'),
            '9': value.get('source'),
        } for value in rest
    ]


Instruction:
Implement a Python 3 function for
populating the 242 MARC field.

Response:
def title_translations2marc(self, key, value):
    """Populate the ``242`` MARC field."""
    return {
        'a': value.get('title'),
        'b': value.get('subtitle'),
        '9': value.get('source'),
    }


Instruction:
Implement a function in Python 3 to
populate the imprints key.

Response:
def imprints(self, key, value):
    """Populate the ``imprints`` key."""
    return {
        'place': value.get('a'),
        'publisher': value.get('b'),
        'date': normalize_date_aggressively(value.get('c')),
    }


Instruction:
Can you write a function in Python 3 where it
populates the 260 MARC field.

Response:
def imprints2marc(self, key, value):
    """Populate the ``260`` MARC field."""
    return {
        'a': value.get('place'),
        'b': value.get('publisher'),
        'c': value.get('date'),
    }


Instruction:
Can you generate the documentation for the following Python 3 function
def public_notes(self, key, value):
    def _means_not_curated(public_note):
        return public_note in [
            '*Brief entry*',
            '* Brief entry *',
            '*Temporary entry*',
            '* Temporary entry *',
            '*Temporary record*',
            '* Temporary record *',
        ]

    public_notes = self.get('public_notes', [])
    thesis_info = self.get('thesis_info', {})

    source = force_single_element(value.get('9', ''))
    for value in force_list(value):
        for public_note in force_list(value.get('a')):
            match = IS_DEFENSE_DATE.match(public_note)
            if match:
                try:
                    thesis_info['defense_date'] = normalize_date(match.group('defense_date'))
                except ValueError:
                    public_notes.append({
                        'source': source,
                        'value': public_note,
                    })
            elif _means_not_curated(public_note):
                self['curated'] = False
            else:
                public_notes.append({
                    'source': source,
                    'value': public_note,
                })

    self['thesis_info'] = thesis_info
    return public_notes

Response: Populate the public_notes key.


Instruction:
Write a Python 3 function that can
populate the thesis_info key.

Response:
def thesis_info(self, key, value):
    """Populate the ``thesis_info`` key."""
    def _get_degree_type(value):
        DEGREE_TYPES_MAP = {
            'RAPPORT DE STAGE': 'other',
            'INTERNSHIP REPORT': 'other',
            'DIPLOMA': 'diploma',
            'BACHELOR': 'bachelor',
            'LAUREA': 'laurea',
            'MASTER': 'master',
            'THESIS': 'other',
            'PHD': 'phd',
            'PDF': 'phd',
            'PH.D. THESIS': 'phd',
            'HABILITATION': 'habilitation',
        }

        b_value = force_single_element(value.get('b', ''))
        if b_value:
            return DEGREE_TYPES_MAP.get(b_value.upper(), 'other')

    def _get_institutions(value):
        c_values = force_list(value.get('c'))
        z_values = force_list(value.get('z'))

        # XXX: we zip only when they have the same length, otherwise
        #      we might match a value with the wrong recid.
        if len(c_values) != len(z_values):
            return [{'name': c_value} for c_value in c_values]
        else:
            return [{
                'curated_relation': True,
                'name': c_value,
                'record': get_record_ref(z_value, 'institutions'),
            } for c_value, z_value in zip(c_values, z_values)]

    thesis_info = self.get('thesis_info', {})

    thesis_info['date'] = normalize_date(force_single_element(value.get('d')))
    thesis_info['degree_type'] = _get_degree_type(value)
    thesis_info['institutions'] = _get_institutions(value)

    return thesis_info


Instruction:
How would you implement a function in Python 3 that
populates the 502 MARC field.

Response:
def thesis_info2marc(self, key, value):
    """Populate the ``502`` MARC field.

    Also populates the ``500`` MARC field through side effects.
    """
    def _get_b_value(value):
        DEGREE_TYPES_MAP = {
            'bachelor': 'Bachelor',
            'diploma': 'Diploma',
            'habilitation': 'Habilitation',
            'laurea': 'Laurea',
            'master': 'Master',
            'other': 'Thesis',
            'phd': 'PhD',
        }

        degree_type = value.get('degree_type')
        if degree_type:
            return DEGREE_TYPES_MAP.get(degree_type)

    result_500 = self.get('500', [])
    result_502 = self.get('502', {})

    if value.get('defense_date'):
        result_500.append({
            'a': u'Presented on {}'.format(value.get('defense_date')),
        })

    result_502 = {
        'b': _get_b_value(value),
        'c': [el['name'] for el in force_list(value.get('institutions'))],
        'd': value.get('date'),
    }

    self['500'] = result_500
    return result_502


Instruction:
Write a Python 3 script for
populating the abstracts key.

Response:
def abstracts(self, key, value):
    """Populate the ``abstracts`` key."""
    result = []

    source = force_single_element(value.get('9'))

    for a_value in force_list(value.get('a')):
        result.append({
            'source': source,
            'value': a_value,
        })

    return result


Instruction:
Given the following Python 3 function, write the documentation
def funding_info(self, key, value):
    return {
        'agency': value.get('a'),
        'grant_number': value.get('c'),
        'project_number': value.get('f'),
    }

Response: Populate the funding_info key.


Instruction:
Here you have a function in Python 3, explain what it does
def funding_info2marc(self, key, value):
    return {
        'a': value.get('agency'),
        'c': value.get('grant_number'),
        'f': value.get('project_number'),
    }

Response: Populate the 536 MARC field.


Instruction:
Write a Python 3 script to
populate the license key.

Response:
def license(self, key, value):
    """Populate the ``license`` key."""
    def _get_license(value):
        a_values = force_list(value.get('a'))

        oa_licenses = [el for el in a_values if el == 'OA' or el == 'Open Access']
        other_licenses = [el for el in a_values if el != 'OA' and el != 'Open Access']

        if not other_licenses:
            return force_single_element(oa_licenses)
        return force_single_element(other_licenses)

    def _get_material(value):
        material = value.get('3', '').lower()
        if material == 'article':
            return 'publication'
        return material

    return {
        'imposing': value.get('b'),
        'license': _get_license(value),
        'material': _get_material(value),
        'url': value.get('u'),
    }


Instruction:
Create a Python 3 function to
populate the 595 MARC field.

Response:
def license2marc(self, key, value):
    """Populate the ``540`` MARC field."""
    return {
        'a': value.get('license'),
        'b': value.get('imposing'),
        'u': value.get('url'),
        '3': value.get('material'),
    }


Instruction:
Can you generate a brief explanation for the following Python 3 code
def copyright(self, key, value):
    MATERIAL_MAP = {
        'Article': 'publication',
        'Published thesis as a book': 'publication',
    }

    material = value.get('e') or value.get('3')

    return {
        'holder': value.get('d'),
        'material': MATERIAL_MAP.get(material),
        'statement': value.get('f'),
        'url': value.get('u'),
        'year': maybe_int(value.get('g')),
    }

Response: Populate the copyright key.


Instruction:
Make a summary of the following Python 3 code
def copyright2marc(self, key, value):
    E_MAP = {
        'publication': 'Article',
    }

    e_value = value.get('material')

    return {
        'd': value.get('holder'),
        'e': E_MAP.get(e_value),
        'f': value.get('statement'),
        'g': value.get('year'),
        'u': value.get('url'),
    }

Response: Populate the 595 MARC field.


Instruction:
Given the following Python 3 function, write the documentation
def _private_notes(self, key, value):
    def _is_for_cds(value):
        normalized_c_values = [el.upper() for el in force_list(value.get('c'))]
        return 'CDS' in normalized_c_values

    def _is_for_hal(value):
        normalized_c_values = [el.upper() for el in force_list(value.get('c'))]
        return 'HAL' in normalized_c_values

    def _is_not_for_hal(value):
        normalized_c_values = [el.upper() for el in force_list(value.get('c'))]
        return 'NOT HAL' in normalized_c_values

    _private_notes = self.get('_private_notes', [])
    _export_to = self.get('_export_to', {})

    for value in force_list(value):
        if _is_for_cds(value):
            _export_to['CDS'] = True

        if _is_for_hal(value):
            _export_to['HAL'] = True
        elif _is_not_for_hal(value):
            _export_to['HAL'] = False

        source = force_single_element(value.get('9'))
        for _private_note in force_list(value.get('a')):
            _private_notes.append({
                'source': source,
                'value': _private_note,
            })

    self['_export_to'] = _export_to
    return _private_notes

Response: Populate the _private_notes key.


Instruction:
How would you explain what the following Python 3 function does
def _private_notes2marc(self, key, value):
    def _is_from_hal(value):
        return value.get('source') == 'HAL'

    if not _is_from_hal(value):
        return {
            '9': value.get('source'),
            'a': value.get('value'),
        }

    self.setdefault('595_H', []).append({'a': value.get('value')})

Response: Populate the 695 MARC key.


Instruction:
Write a Python 3 script to
populate the 595 MARC field.

Response:
def _export_to2marc(self, key, value):
    """Populate the ``595`` MARC field."""
    def _is_for_cds(value):
        return 'CDS' in value

    def _is_for_hal(value):
        return 'HAL' in value and value['HAL']

    def _is_not_for_hal(value):
        return 'HAL' in value and not value['HAL']

    result = []

    if _is_for_cds(value):
        result.append({'c': 'CDS'})

    if _is_for_hal(value):
        result.append({'c': 'HAL'})
    elif _is_not_for_hal(value):
        result.append({'c': 'not HAL'})

    return result


Instruction:
Create a Python 3 function for
populating the _desy_bookkeeping key.

Response:
def _desy_bookkeeping(self, key, value):
    """Populate the ``_desy_bookkeeping`` key."""
    return {
        'date': normalize_date(value.get('d')),
        'expert': force_single_element(value.get('a')),
        'status': value.get('s'),
    }


Instruction:
Given the following Python 3 function, write the documentation
def _desy_bookkeeping2marc(self, key, value):
    if 'identifier' not in value:
        return {
            'a': value.get('expert'),
            'd': value.get('date'),
            's': value.get('status'),
        }

    self.setdefault('035', []).append({
        '9': 'DESY',
        'z': value['identifier']
    })

Response: Populate the 595_D MARC field.


Instruction:
How would you explain what the following Python 3 function does
def _dates(self, key, value):
    if value.get('q'):
        self['date_proposed'] = normalize_date(value['q'])
    if value.get('r'):
        self['date_approved'] = normalize_date(value['r'])
    if value.get('s'):
        self['date_started'] = normalize_date(value['s'])
    if value.get('c'):
        self['date_cancelled'] = normalize_date(value['c'])
    if value.get('t'):
        self['date_completed'] = normalize_date(value['t'])

    raise IgnoreKey

Response: Populate the dates key.


Instruction:
Write a Python 3 function that can
populate the experiment key.

Response:
def experiment(self, key, values):
    """Populate the ``experiment`` key.

    Also populates the ``legacy_name``, the ``accelerator``, and the
    ``institutions`` keys through side effects.
    """
    experiment = self.get('experiment', {})
    legacy_name = self.get('legacy_name', '')
    accelerator = self.get('accelerator', {})
    institutions = self.get('institutions', [])

    for value in force_list(values):
        if value.get('c'):
            experiment['value'] = value.get('c')
        if value.get('d'):
            experiment['short_name'] = value.get('d')

        if value.get('a'):
            legacy_name = value.get('a')

        if value.get('b'):
            accelerator['value'] = value.get('b')

        institution = {}
        if value.get('u'):
            institution['value'] = value.get('u')
        if value.get('z'):
            record = get_record_ref(maybe_int(value.get('z')), 'institutions')
            if record:
                institution['curated_relation'] = True
                institution['record'] = record
        institutions.append(institution)

    self['legacy_name'] = legacy_name
    self['accelerator'] = accelerator
    self['institutions'] = institutions
    return experiment


Instruction:
Can you implement a function in Python 3 that
populates the core key.

Response:
def core(self, key, value):
    """Populate the ``core`` key.

    Also populates the ``deleted`` and ``project_type`` keys through side
    effects.
    """
    core = self.get('core')
    deleted = self.get('deleted')
    project_type = self.get('project_type', [])

    if not core:
        normalized_a_values = [el.upper() for el in force_list(value.get('a'))]
        if 'CORE' in normalized_a_values:
            core = True

    if not deleted:
        normalized_c_values = [el.upper() for el in force_list(value.get('c'))]
        if 'DELETED' in normalized_c_values:
            deleted = True

    if not project_type:
        normalized_a_values = [el.upper() for el in force_list(value.get('a'))]
        if 'ACCELERATOR' in normalized_a_values:
            project_type.append('accelerator')

    self['project_type'] = project_type
    self['deleted'] = deleted
    return core


Instruction:
Create a Python 3 function for
populating the control_number key.

Response:
def control_number(endpoint):
    """Populate the ``control_number`` key.

    Also populates the ``self`` key through side effects.
    """
    def _control_number(self, key, value):
        self['self'] = get_record_ref(int(value), endpoint)
        return int(value)

    return _control_number


Instruction:
Explain what the following Python 3 code does
def acquisition_source(self, key, value):
    def _get_datetime(value):
        d_value = force_single_element(value.get('d', ''))
        if d_value:
            try:
                date = PartialDate.loads(d_value)
            except ValueError:
                return d_value
            else:
                datetime_ = datetime(year=date.year, month=date.month, day=date.day)
                return datetime_.isoformat()

    internal_uid, orcid, source = None, None, None

    a_values = force_list(value.get('a'))
    for a_value in a_values:
        if IS_INTERNAL_UID.match(a_value):
            if a_value.startswith('inspire:uid:'):
                internal_uid = int(a_value[12:])
            else:
                internal_uid = int(a_value)
        elif IS_ORCID.match(a_value):
            if a_value.startswith('orcid:'):
                orcid = a_value[6:]
            else:
                orcid = a_value
        else:
            source = a_value

    c_value = force_single_element(value.get('c', ''))
    normalized_c_value = c_value.lower()

    if normalized_c_value == 'batchupload':
        method = 'batchuploader'
    elif normalized_c_value == 'submission':
        method = 'submitter'
    else:
        method = normalized_c_value

    return {
        'datetime': _get_datetime(value),
        'email': value.get('b'),
        'internal_uid': internal_uid,
        'method': method,
        'orcid': orcid,
        'source': source,
        'submission_number': value.get('e'),
    }

Response: Populate the acquisition_source key.


Instruction:
Create a Python 3 function for
populating the public_notes key.

Response:
def public_notes_500(self, key, value):
    """Populate the ``public_notes`` key."""
    return [
        {
            'source': value.get('9'),
            'value': public_note,
        } for public_note in force_list(value.get('a'))
    ]


Instruction:
Here you have a function in Python 3, explain what it does
def _private_notes_595(self, key, value):
    return [
        {
            'source': value.get('9'),
            'value': _private_note,
        } for _private_note in force_list(value.get('a'))
    ]

Response: Populate the _private_notes key.


Instruction:
Here you have a function in Python 3, explain what it does
def external_system_identifiers(endpoint):
    @utils.flatten
    @utils.for_each_value
    def _external_system_identifiers(self, key, value):
        new_recid = maybe_int(value.get('d'))
        if new_recid:
            self['new_record'] = get_record_ref(new_recid, endpoint)

        return [
            {
                'schema': 'SPIRES',
                'value': ext_sys_id,
            } for ext_sys_id in force_list(value.get('a'))
        ]

    return _external_system_identifiers

Response: Populate the external_system_identifiers key.


Instruction:
Can you implement a function in Python 3 that
populates the deleted_records key.

Response:
def deleted_records(endpoint):
    """Populate the ``deleted_records`` key."""
    @utils.for_each_value
    def _deleted_records(self, key, value):
        deleted_recid = maybe_int(value.get('a'))
        if deleted_recid:
            return get_record_ref(deleted_recid, endpoint)

    return _deleted_records


Instruction:
Given the following Python 3 function, write the documentation
def accelerator_experiments(self, key, value):
    result = []

    a_value = force_single_element(value.get('a'))
    e_values = [el for el in force_list(value.get('e')) if el != '-']
    zero_values = force_list(value.get('0'))

    if a_value and not e_values:
        result.append({'accelerator': a_value})

    # XXX: we zip only when they have the same length, otherwise
    #      we might match a value with the wrong recid.
    if len(e_values) == len(zero_values):
        for e_value, zero_value in zip(e_values, zero_values):
            result.append({
                'legacy_name': e_value,
                'record': get_record_ref(zero_value, 'experiments'),
            })
    else:
        for e_value in e_values:
            result.append({'legacy_name': e_value})

    return result

Response: Populate the accelerator_experiments key.


Instruction:
Make a summary of the following Python 3 code
def keywords(self, key, values):
    keywords = self.get('keywords', [])
    energy_ranges = self.get('energy_ranges', [])

    for value in force_list(values):
        if value.get('a'):
            schema = force_single_element(value.get('2', '')).upper()
            sources = force_list(value.get('9'))

            a_values = force_list(value.get('a'))

            if 'conference' not in sources:
                for a_value in a_values:
                    keywords.append({
                        'schema': schema,
                        'source': force_single_element(sources),
                        'value': a_value,
                    })

        if value.get('e'):
            energy_ranges.append(ENERGY_RANGES_MAP.get(value.get('e')))

    self['energy_ranges'] = energy_ranges
    return keywords

Response: Populate the keywords key.


Instruction:
How would you explain what the following Python 3 function does
def keywords2marc(self, key, values):
    result_695 = self.get('695', [])
    result_084 = self.get('084', [])
    result_6531 = self.get('6531', [])

    for value in values:
        schema = value.get('schema')
        source = value.get('source')
        keyword = value.get('value')

        if schema == 'PACS' or schema == 'PDG':
            result_084.append({
                '2': schema,
                '9': source,
                'a': keyword,
            })
        elif schema == 'JACOW':
            result_6531.append({
                '2': 'JACoW',
                '9': source,
                'a': keyword,
            })
        elif schema == 'INSPIRE':
            result_695.append({
                '2': 'INSPIRE',
                '9': source,
                'a': keyword,
            })
        elif schema == 'INIS':
            result_695.append({
                '2': 'INIS',
                '9': source,
                'a': keyword,
            })
        elif source != 'magpie':
            result_6531.append({
                '9': source,
                'a': keyword,
            })

    self['6531'] = result_6531
    self['084'] = result_084
    return result_695

Response: Populate the 695 MARC field.


Instruction:
Can you generate the documentation for the following Python 3 function
def collaborations(self, key, value):
    result = []

    for g_value in force_list(value.get('g')):
        collaborations = normalize_collaboration(g_value)
        if len(collaborations) == 1:
            result.append({
                'record': get_record_ref(maybe_int(value.get('0')), 'experiments'),
                'value': collaborations[0],
            })
        else:
            result.extend({'value': collaboration} for collaboration in collaborations)

    return result

Response: Populate the collaborations key.


Instruction:
How would you explain what the following Python 3 function does
def publication_info(self, key, value):
    def _get_cnum(value):
        w_value = force_single_element(value.get('w', ''))
        normalized_w_value = w_value.replace('/', '-').upper()

        return normalized_w_value

    def _get_material(value):
        schema = load_schema('elements/material')
        valid_materials = schema['enum']

        m_value = force_single_element(value.get('m', ''))
        normalized_m_value = m_value.lower()

        if normalized_m_value in valid_materials:
            return normalized_m_value

    def _get_parent_isbn(value):
        z_value = force_single_element(value.get('z', ''))
        if z_value:
            return normalize_isbn(z_value)

    def _get_pubinfo_freetext(value):
        x_value = force_single_element(value.get('x', ''))
        if not x_value.startswith('#DONE'):
            return x_value

    page_start, page_end, artid = split_page_artid(value.get('c'))

    parent_recid = maybe_int(force_single_element(value.get('0')))
    parent_record = get_record_ref(parent_recid, 'literature')

    journal_recid = maybe_int(force_single_element(value.get('1')))
    journal_record = get_record_ref(journal_recid, 'journals')

    conference_recid = maybe_int(force_single_element(value.get('2')))
    conference_record = get_record_ref(conference_recid, 'conferences')

    return {
        'artid': artid,
        'cnum': _get_cnum(value),
        'conf_acronym': force_single_element(value.get('q')),
        'conference_record': conference_record,
        'hidden': key.startswith('7731') or None,
        'journal_issue': force_single_element(value.get('n')),
        'journal_record': journal_record,
        'journal_title': force_single_element(value.get('p')),
        'journal_volume': force_single_element(value.get('v')),
        'material': _get_material(value),
        'page_end': page_end,
        'page_start': page_start,
        'parent_isbn': _get_parent_isbn(value),
        'parent_record': parent_record,
        'parent_report_number': force_single_element(value.get('r')),
        'pubinfo_freetext': _get_pubinfo_freetext(value),
        'year': maybe_int(force_single_element(value.get('y'))),
    }

Response: Populate the publication_info key.


Instruction:
Make a summary of the following Python 3 code
def publication_info2marc(self, key, values):
    result_773 = self.get('773', [])
    result_7731 = self.get('7731', [])

    for value in force_list(convert_new_publication_info_to_old(values)):
        page_artid = []
        if value.get('page_start') and value.get('page_end'):
            page_artid.append(u'{page_start}-{page_end}'.format(**value))
        elif value.get('page_start'):
            page_artid.append(u'{page_start}'.format(**value))
        elif value.get('artid'):
            page_artid.append(u'{artid}'.format(**value))

        result = {
            '0': get_recid_from_ref(value.get('parent_record')),
            'c': page_artid,
            'm': value.get('material'),
            'n': value.get('journal_issue'),
            'p': value.get('journal_title'),
            'q': value.get('conf_acronym'),
            'r': value.get('parent_report_number'),
            'v': value.get('journal_volume'),
            'w': value.get('cnum'),
            'x': value.get('pubinfo_freetext'),
            'y': value.get('year'),
            'z': value.get('parent_isbn'),
        }

        if value.get('hidden'):
            result_7731.append(result)
        else:
            result_773.append(result)

    self['7731'] = result_7731
    return result_773

Response: Populate the 773 MARC field.


Instruction:
Can you tell what is the following Python 3 function doing
def related_records_78002(self, key, value):
    record = get_record_ref(maybe_int(value.get('w')), 'literature')
    if record:
        return {
            'curated_relation': record is not None,
            'record': record,
            'relation': 'predecessor',
        }

Response: Populate the related_records key.


Instruction:
Can you generate the documentation for the following Python 3 function
def related_records_78502(self, key, value):
    record = get_record_ref(maybe_int(value.get('w')), 'literature')
    if record:
        return {
            'curated_relation': record is not None,
            'record': record,
            'relation': 'successor',
        }

Response: Populate the related_records key.


Instruction:
Create a Python 3 function for
populating the related_records key.

Response:
def related_records_78708(self, key, value):
    """Populate the ``related_records`` key."""
    record = get_record_ref(maybe_int(value.get('w')), 'literature')
    if record:
        return {
            'curated_relation': record is not None,
            'record': record,
            'relation_freetext': value.get('i'),
        }


Instruction:
Create a Python 3 function for
populating the 78502 MARC field based on the related_records key.

Response:
def related_records2marc(self, key, value):
    """Populate the ``78708`` MARC field

    Also populates the ``78002``, ``78502`` MARC fields through side effects.
    """
    if value.get('relation_freetext'):
        return {
            'i': value.get('relation_freetext'),
            'w': get_recid_from_ref(value.get('record')),
        }
    elif value.get('relation') == 'successor':
        self.setdefault('78502', []).append({
            'i': 'superseded by',
            'w': get_recid_from_ref(value.get('record')),
        })
    elif value.get('relation') == 'predecessor':
        self.setdefault('78002', []).append({
            'i': 'supersedes',
            'w': get_recid_from_ref(value.get('record')),
        })
    else:
        raise NotImplementedError(u"Unhandled relation in related_records: {}".format(value.get('relation')))


Instruction:
Can you implement a function in Python 3 that
populates the _private_notes key.

Response:
def _private_notes(self, key, value):
    """Populate the ``_private_notes`` key."""
    return [
        {
            'source': value.get('9'),
            'value': _private_note,
        } for _private_note in force_list(value.get('x'))
    ]


Instruction:
Write a Python 3 function that can
populate the proceedings key.

Response:
def proceedings(self, key, value):
    """Populate the ``proceedings`` key.

    Also populates the ``refereed`` key through side effects.
    """
    proceedings = self.get('proceedings')
    refereed = self.get('refereed')

    if not proceedings:
        normalized_a_values = [el.upper() for el in force_list(value.get('a'))]
        if 'PROCEEDINGS' in normalized_a_values:
            proceedings = True

    if not refereed:
        normalized_a_values = [el.upper() for el in force_list(value.get('a'))]
        if 'PEER REVIEW' in normalized_a_values:
            refereed = True
        elif 'NON-PUBLISHED' in normalized_a_values:
            refereed = False

    self['refereed'] = refereed
    return proceedings


Instruction:
Implement a function in Python 3 to
populate the short_title key.

Response:
def short_title(self, key, value):
    """Populate the ``short_title`` key.

    Also populates the ``title_variants`` key through side effects.
    """
    short_title = value.get('a')
    title_variants = self.get('title_variants', [])

    if value.get('u'):
        short_title = value.get('u')
        title_variants.append(value.get('a'))

    self['title_variants'] = title_variants
    return short_title


Instruction:
Write a Python 3 function that can
populate the deleted key.

Response:
def deleted(self, key, value):
    """Populate the ``deleted`` key.

    Also populates the ``book_series`` key through side effects.
    """
    deleted = self.get('deleted')
    book_series = self.get('book_series')

    if not deleted:
        normalized_a_values = [el.upper() for el in force_list(value.get('a'))]
        normalized_c_values = [el.upper() for el in force_list(value.get('c'))]
        if 'DELETED' in normalized_a_values or 'DELETED' in normalized_c_values:
            deleted = True

    if not book_series:
        normalized_a_values = [el.upper() for el in force_list(value.get('a'))]
        if 'BOOKSERIES' in normalized_a_values:
            book_series = True

    self['book_series'] = book_series
    return deleted


Instruction:
Make a summary of the following Python 3 code
def ranks(self, key, value):
    return [normalize_rank(el) for el in force_list(value.get('a'))]

Response: Populate the ranks key.


Instruction:
Create a Python 3 function for
creating a command line argument parser

Response:
def new_parser(self):
        """ Create a command line argument parser

        Add a few default flags, such as --version
        for displaying the program version when invoked """

        parser = argparse.ArgumentParser(description=self.description)
        parser.add_argument(
            '--version', help='show version and exit',
            default=False, action='store_true')
        parser.add_argument(
            '--debug', help='enable debugging',
            default=False, action='store_true')
        return parser


Instruction:
Make a summary of the following Python 3 code
def add_command(self, command, function, description=None):

        self.registered[command] = {
            'function': function, 'description': description
        }

Response: Register a new function with a the name command and optionally description.


Instruction:
Write a Python 3 function for
parsing command line arguments if necessary then run program.

Response:
def run(self, args=None):
        """ Parse command line arguments if necessary then run program.
        By default this method will just take of the --version flag.

        The logic for other flags should be handled by your subclass """

        args = args or self.parser.parse_args()

        if args.debug:
            logging.basicConfig(level=logging.DEBUG)

        if args.version:
            print(version.VERSION)
            sys.exit(0)


Instruction:
Write a Python 3 function for
showing help for all available commands or just a single one

Response:
def help_function(self, command=None):
        """ Show help for all available commands or just a single one """
        if command:
            return self.registered[command].get(
                'description', 'No help available'
            )
        return ', '.join(sorted(self.registered))


Instruction:
Can you create a Python 3 function that
registers a new function for command

Response:
def add_command(self, command, function, description=None):
        """ Register a new function for command """
        super(Program, self).add_command(command, function, description)
        self.service.register(command, function)


Instruction:
Create a Python 3 function to
parse comand line arguments and run program

Response:
def run(self, args=None):
        """ Parse comand line arguments/flags and run program """

        args = args or self.parser.parse_args()
        super(Program, self).run(args)

        # Read configuration file if any
        if args.config is not None:
            filepath = args.config
            self.config.read(filepath)

        # Start workers then wait until they finish work
        [w.start() for w in self.workers]
        [w.join() for w in self.workers]


Instruction:
Can you create a Python 3 function that
creates a client based on addr

Response:
def create_client(self, addr, timeout):
        """ Create client(s) based on addr """

        def make(addr):
            c = Client(addr)
            c.socket._set_recv_timeout(timeout)
            return c

        if ',' in addr:
            addrs = addr.split(',')
            addrs = [a.strip() for a in addrs]
            return {a: make(a) for a in addrs}
        return make(addr)


Instruction:
How would you explain what the following Python 3 function does
def _call_single(self, client, command, *args):
        try:
            return client.call(command, *args)
        except Exception as e:
            return None, str(e)

Response: Call a single command and return the result.


Instruction:
Create a Python 3 function to
call multiple commands on a set of resources.

Response:
def _call_multi(self, clients, command, *args):
        """ Call multi """
        responses, errors = {}, {}
        for addr, client in clients.items():
            res, err = self._call_single(client, command, *args)
            responses[addr] = res
            errors[addr] = err
        return responses, errors


Instruction:
Write a Python 3 function that can
call a remote service.

Response:
def call(self, command, *args):
        """ Call remote service(s) """
        if isinstance(self.c, dict):
            return self._call_multi(self.c, command, *args)
        return self._call_single(self.c, command, *args)


Instruction:
Can you generate the documentation for the following Python 3 function
def close(self):
        if isinstance(self.c, dict):
            for client in self.c.values():
                client.sock.close()
            return
        self.c.socket.close()

Response: Close socket and all its related resources.


Instruction:
Create a Python 3 function to
show result or error

Response:
def _show(self, res, err, prefix='', colored=False):
        """ Show result or error """

        if self.kind is 'local':
            what = res if not err else err
            print(what)
            return

        if self.kind is 'remote':
            if colored:
                red, green, reset = Fore.RED, Fore.GREEN, Fore.RESET
            else:
                red = green = reset = ''
            if err:
                what = prefix + red + 'remote err: {}'.format(err) + reset
            else:
                what = prefix + green + str(res) + reset
            print(what)


Instruction:
Can you create a Python 3 function that
executes local or remote command and show response

Response:
def call(self, command, *args):
        """ Execute local OR remote command and show response """

        if not command:
            return

        # Look for local methods first
        try:
            res = self.registered[command]['function'](self, *args)
            return Response('local', res, None)

        # Method not found, try remote
        except KeyError:

            # Execute remote command
            res, err = self.client.call(command, *args)
            return Response('remote', res, err, self.client.is_multi())

        # Local exception
        except Exception as e:
            return Response('local', res, str(e))


Instruction:
Create a Python 3 function for
parsing ctl user input. Double quotes are used to group together multi words arguments.

Response:
def parse_input(self, text):
        """ Parse ctl user input. Double quotes are used
        to group together multi words arguments. """

        parts = util.split(text)
        command = parts[0] if text and parts else None
        command = command.lower() if command else None
        args = parts[1:] if len(parts) > 1 else []

        return (command, args)


Instruction:
Implement a Python 3 function for
entering loop read user input then run command. Repeat

Response:
def loop(self):
        """ Enter loop, read user input then run command. Repeat """

        while True:
            text = compat.input('ctl > ')
            command, args = self.parse_input(text)
            if not command:
                continue
            response = self.call(command, *args)
            response.show()


Instruction:
Create a Python 3 function for
reversing namespace lookup. Note that returned namespace may not be unique :param uri: namespace URI :return: namespace

Response:
def namespace_for(uri: Union[URIRef, Namespace, str]) -> str:
    """
    Reverse namespace lookup.  Note that returned namespace may not be unique
    :param uri: namespace URI
    :return: namespace
    """
    uri = str(uri)
    if uri not in namespaces.values():
        namespaces[AnonNS().ns] = uri
    return [k for k, v in namespaces.items() if uri == v][0]


Instruction:
How would you explain what the following Python 3 function does
def split(text):

    # Cleanup text
    text = text.strip()
    text = re.sub('\s+', ' ', text)  # collpse multiple spaces

    space, quote, parts = ' ', '"', []
    part, quoted = '', False

    for char in text:

        # Encoutered beginning double quote
        if char is quote and quoted is False:
            quoted = True
            continue

        # Encountered the ending double quote
        if char is quote and quoted is True:
            quoted = False
            parts.append(part.strip())
            part = ''
            continue

        # Found space in quoted
        if char is space and quoted is True:
            part += char
            continue

        # Found space but not quoted
        if char is space:
            if part:
                parts.append(part)
                part = ''
            continue

        # Found other character
        if char is not space:
            part += char
            continue

    if part:
        parts.append(part.strip())

    return parts

Response: Split text into arguments accounting for muti - word arguments


Instruction:
Implement a function in Python 3 to
return a list of all available formats in rdflib for the required task

Response:
def known_formats(use: Union[Serializer, Parser]=Serializer, include_mime_types: bool = False) -> List[str]:
    """ Return a list of available formats in rdflib for the required task
    :param use: task (typically Serializer or Parser)
    :param include_mime_types: whether mime types are included in the return list
    :return: list of formats
    """
    return sorted([name for name, kind in plugin._plugins.keys()
                   if kind == use and (include_mime_types or '/' not in name)])


Instruction:
How would you code a function in Python 3 to
return the signature for a file name.

Response:
def file_signature(file_name: str) -> Optional[Tuple]:
    """
    Return an identity signature for file name
    :param file_name: name of file
    :return: mode, size, last modified time if file exists, otherwise none
    """
    try:
        st = os.stat(file_name)
    except FileNotFoundError:
        return None
    return stat.S_IFMT(st.st_mode), st.st_size, st.st_mtime


Instruction:
Can you generate a brief explanation for the following Python 3 code
def url_signature(url: str) -> Optional[Tuple]:
    request = urllib.request.Request(url)
    request.get_method = lambda: 'HEAD'
    response = None
    try:
        response = urllib.request.urlopen(request)
    except urllib.error.HTTPError:
        return None
    return response.info()['Last-Modified'], response.info()['Content-Length'], response.info().get('ETag')

Response: Return an identify signature for a given URL.


Instruction:
Can you generate the documentation for the following Python 3 function
def signature(name: str) -> Optional[Tuple]:
    return url_signature(name) if is_url(name) else file_signature(name) if is_file(name) else None

Response: Returns the file or URL signature for name


Instruction:
How would you explain what the following Python 3 function does
def read_long_description(readme_file):
    try:
        import pypandoc
    except (ImportError, OSError) as e:
        print('No pypandoc or pandoc: %s' % (e,))
        if is_py3:
            fh = open(readme_file, encoding='utf-8')
        else:
            fh = open(readme_file)
        long_description = fh.read()
        fh.close()
        return long_description
    else:
        return pypandoc.convert(readme_file, 'rst')

Response: Read package long description from README file


Instruction:
Create a Python 3 function to
remove the prefixes from the graph for aesthetics

Response:
def strip_prefixes(g: Graph):
        """ Remove the prefixes from the graph for aesthetics """
        return re.sub(r'^@prefix .* .\n', '',
                      g.serialize(format="turtle").decode(),
                      flags=re.MULTILINE).strip()


Instruction:
Make a summary of the following Python 3 code
def add_prefixes(self, nsmap: Dict[str, Namespace]) -> None:
        [self._g.bind(e[0], e[1]) for e in nsmap.items()]

Response: Add the required prefix definitions


Instruction:
Can you generate the documentation for the following Python 3 function
def add(self, subj: Node, pred: URIRef, obj: Node) -> "FHIRResource":
        self._g.add((subj, pred, obj))
        return self

Response: Shortcut to rdflib add function


Instruction:
Write a Python 3 script for
adding a value node to the object.

Response:
def add_value_node(self, subj: Node, pred: URIRef, val: Union[JsonObj, str, List],
                       valuetype: Optional[URIRef]= None) -> None:
        """
        Expand val according to the range of pred and add it to the graph
        :param subj: graph subject
        :param pred: graph predicate
        :param val: JSON representation of target object
        :param valuetype: predicate type if it can't be directly determined
        """
        pred_type = self._meta.predicate_type(pred) if not valuetype else valuetype
        # Transform generic resources into specific types
        if pred_type == FHIR.Resource:
            pred_type = FHIR[val.resourceType]

        val_meta = FHIRMetaVocEntry(self._vocabulary, pred_type)
        for k, p in val_meta.predicates().items():
            if k in val:
                self.add_val(subj, p, val, k)
                if pred == FHIR.CodeableConcept.coding:
                    self.add_type_arc(subj, val)
            elif k == "value" and val_meta.predicate_type(p) == FHIR.Element:
                # value / Element is the wild card combination -- if there is a "value[x]" in val, emit it where the
                # type comes from 'x'
                for vk in val._as_dict.keys():
                    if vk.startswith(k):
                        self.add_val(subj, FHIR['Extension.' + vk], val, vk, self._meta.value_predicate_to_type(vk))
            else:
                # Can have an extension only without a primary value
                self.add_extension_val(subj, val, k, p)


Instruction:
Can you generate the documentation for the following Python 3 function
def add_reference(self, subj: Node, val: str) -> None:
        match = FHIR_RESOURCE_RE.match(val)
        ref_uri_str = res_type = None
        if match:
            ref_uri_str = val if match.group(FHIR_RE_BASE) else (self._base_uri + urllib.parse.quote(val))
            res_type = match.group(FHIR_RE_RESOURCE)
        elif '://' in val:
            ref_uri_str = val
            res_type = "Resource"
        elif self._base_uri and not val.startswith('#') and not val.startswith('/'):
            ref_uri_str = self._base_uri + urllib.parse.quote(val)
            res_type = val.split('/', 1)[0] if '/' in val else "Resource"
        if ref_uri_str:
            ref_uri = URIRef(ref_uri_str)
            self.add(subj, FHIR.link, ref_uri)
            self.add(ref_uri, RDF.type, FHIR[res_type])

Response: Add a fhir : link and RDF type arc if it can be determined


Instruction:
Here you have a function in Python 3, explain what it does
def add_val(self, subj: Node, pred: URIRef, json_obj: JsonObj, json_key: str,
                valuetype: Optional[URIRef] = None) -> Optional[BNode]:
        """
        Add the RDF representation of val to the graph as a target of subj, pred.  Note that FHIR lists are
        represented as a list of BNODE objects with a fhir:index discrimanant
        :param subj: graph subject
        :param pred: predicate
        :param json_obj: object containing json_key
        :param json_key: name of the value in the JSON resource
        :param valuetype: value type if NOT determinable by predicate
        :return: value node if target is a BNode else None
        """
        if json_key not in json_obj:
            print("Expecting to find object named '{}' in JSON:".format(json_key))
            print(json_obj._as_json_dumps())
            print("entry skipped")
            return None
        val = json_obj[json_key]
        if isinstance(val, List):
            list_idx = 0
            for lv in val:
                entry_bnode = BNode()
                # TODO: this is getting messy. Refactor and clean this up
                if pred == FHIR.Bundle.entry:
                    entry_subj = URIRef(lv.fullUrl)
                    self.add(entry_bnode, FHIR.index, Literal(list_idx))
                    self.add_val(entry_bnode, FHIR.Bundle.entry.fullUrl, lv, 'fullUrl')
                    self.add(entry_bnode, FHIR.Bundle.entry.resource, entry_subj)
                    self.add(subj, pred, entry_bnode)
                    entry_mv = FHIRMetaVocEntry(self._vocabulary, FHIR.BundleEntryComponent)
                    for k, p in entry_mv.predicates().items():
                        if k not in ['resource', 'fullUrl'] and k in lv:
                            print("---> adding {}".format(k))
                            self.add_val(subj, p, lv, k)
                    FHIRResource(self._vocabulary, None,  self._base_uri, lv.resource, self._g,
                                 False, self._replace_narrative_text, False, resource_uri=entry_subj)
                else:
                    self.add(entry_bnode, FHIR.index, Literal(list_idx))
                    if isinstance(lv, JsonObj):
                        self.add_value_node(entry_bnode, pred, lv, valuetype)
                    else:
                        vt = self._meta.predicate_type(pred)
                        atom_type = self._meta.primitive_datatype_nostring(vt) if vt else None
                        self.add(entry_bnode, FHIR.value, Literal(lv, datatype=atom_type))
                    self.add(subj, pred, entry_bnode)
                list_idx += 1
        else:
            vt = self._meta.predicate_type(pred) if not valuetype else valuetype
            if self._meta.is_atom(pred):
                if self._replace_narrative_text and pred == FHIR.Narrative.div and len(val) > 120:
                    val = REPLACED_NARRATIVE_TEXT
                self.add(subj, pred, Literal(val))
            else:
                v = BNode()
                if self._meta.is_primitive(vt):
                    self.add(v, FHIR.value, Literal(str(val), datatype=self._meta.primitive_datatype_nostring(vt, val)))
                else:
                    self.add_value_node(v, pred, val, valuetype)
                self.add(subj, pred, v)
                if pred == FHIR.Reference.reference:
                    self.add_reference(subj, val)
                self.add_extension_val(v, json_obj, json_key)
                return v
        return None

Response: Add the RDF representation of val to the graph as a target of subj pred.


Instruction:
Can you generate the documentation for the following Python 3 function
def add_extension_val(self,
                          subj: Node,
                          json_obj:
                          Union[JsonObj, List[JsonObjTypes]],
                          key: str,
                          pred: Optional[URIRef] = None) -> None:
        """
        Add any extensions for the supplied object. This can be called in following situations:
        1) Single extended value
                "key" : (value),
                "_key" : {
                    "extension": [
                       {
                        "url": "http://...",
                        "value[x]": "......"
                       }
                    ]
                }
        2) Single extension only
                "_key" : {
                    "extension": [
                       {
                        "url": "http://...",
                        "value[x]": "......"
                       }
                    ]
                }
        3) Multiple extended values:
                (TBD)

        4) Multiple extensions only
                "_key" : [
                  {
                    "extension": [
                       {
                        "url": "http://...",
                        "value[x]": "......"
                       }
                    ]
                  }
                ]

        :param subj: Node containing subject
        :param json_obj: Object (potentially) containing "_key"
        :param key: name of element that is possibly extended (as indicated by "_" prefix)
        :param pred: predicate for the contained elements. Only used in situations 3) (?) and 4
        """
        extendee_name = "_" + key
        if extendee_name in json_obj:
            if not isinstance(subj, BNode):
                raise NotImplementedError("Extension to something other than a simple BNode")
            if isinstance(json_obj[extendee_name], list):
                if not pred:
                    raise NotImplemented("Case 3 not implemented")
                entry_idx = 0
                for extension in json_obj[extendee_name]:
                    entry = BNode()
                    self.add(entry, FHIR.index, Literal(entry_idx))
                    self.add_val(entry, FHIR.Element.extension, extension, 'extension')
                    self.add(subj, pred, entry)
                    entry_idx += 1
            elif 'fhir_comments' in json_obj[extendee_name] and len(json_obj[extendee_name]) == 1:
                # TODO: determine whether and how fhir comments should be represented in RDF.
                # for the moment we just drop them
                print("fhir_comment ignored")
                print(json_obj[extendee_name]._as_json_dumps())
                pass
            else:
                self.add_val(subj, FHIR.Element.extension, json_obj[extendee_name], 'extension')

Response: Add any extensions for the supplied object.


Instruction:
Make a summary of the following Python 3 code
def link(g: Graph, subject: Node, predicate: URIRef) -> Tuple[Optional[URIRef], Optional[URIRef]]:
    link_node = g.value(subject, predicate)
    if link_node:
        l = g.value(link_node, FHIR.link)
        if l:
            typ = g.value(l, RDF.type)
            return l, typ
    return None, None

Response: Return the link URI and link type for subject and predicate


Instruction:
Can you generate a brief explanation for the following Python 3 code
def codeable_concept_code(g: Graph, subject: Node, predicate: URIRef, system: Optional[str]=None) \
        -> List[CodeableConcept]:
    """
    Return a list of CodeableConcept entries for the supplied subject and predicate in graph g
    :param g: graph containing the data
    :param subject: subject
    :param predicate: predicate
    :param system: coding system.  If present, only concepts in this system will be returned
    :return: system, code and optional URI of matching concept(s)
    """
    # EXAMPLE:
    # fhir:Patient.maritalStatus [
    #    fhir:CodeableConcept.coding [
    #      fhir:index 0;
    #      a sct:36629006;
    #      fhir:Coding.system [ fhir:value "http://snomed.info/sct" ];
    #      fhir:Coding.code [ fhir:value "36629006" ];
    #      fhir:Coding.display [ fhir:value "Legally married" ]
    #    ], [
    #      fhir:index 1;
    #      fhir:Coding.system [ fhir:value "http://hl7.org/fhir/v3/MaritalStatus" ];
    #      fhir:Coding.code [ fhir:value "M" ]
    #    ]
    # ];
    rval = []
    coded_entry = g.value(subject, predicate, any=False)
    if coded_entry:
        for codeable_concept in list(g.objects(coded_entry, FHIR.CodeableConcept.coding)):
            coding_system = value(g, codeable_concept, FHIR.Coding.system)
            coding_code = value(g, codeable_concept, FHIR.Coding.code)
            if coding_system and coding_code and (system is None or system == coding_system):
                rval.append(CodeableConcept(coding_system, coding_code, g.value(codeable_concept, RDF.type, any=False)))
    return rval

Response: Returns a list of CodeableConcept objects for the supplied subject and predicate in the given RDF graph g.


Instruction:
Make a summary of the following Python 3 code
def _to_str(uri: URIRef) -> str:
        local_name = str(uri).replace(str(FHIR), '')
        return local_name.rsplit('.', 1)[1] if '.' in local_name else local_name

Response: Convert a FHIR style URI into a JSON representation of a tag name.


Instruction:
Implement a Python 3 function for
returning the tag names and corresponding URI s for all properties that can be associated with subject

Response:
def predicates(self) -> Dict[str, URIRef]:
        """
        Return the tag names and corresponding URI's for all properties that can be associated with subject
        :return: Map from tag name (JSON object identifier) to corresponding URI
        """
        rval = dict()
        for parent in self._o.objects(self._subj, RDFS.subClassOf):
            if isinstance(parent, URIRef) and not str(parent).startswith(str(W5)):
                rval.update(**FHIRMetaVocEntry(self._o, parent).predicates())
        for s in self._o.subjects(RDFS.domain, self._subj):
            rval[self._to_str(s)] = s
        return rval


Instruction:
Can you implement a function in Python 3 that
returns the type of predicate pred

Response:
def predicate_type(self, pred: URIRef) -> URIRef:
        """
        Return the type of pred
        :param pred: predicate to map
        :return:
        """
        return self._o.value(pred, RDFS.range)


Instruction:
Create a Python 3 function for
checking if the given FHIR type is valid.

Response:
def is_valid(self, t: URIRef) -> bool:
        """
        Raise an exception if 't' is unrecognized
        :param t: metadata URI
        """
        if not self.has_type(t):
            raise TypeError("Unrecognized FHIR type: {}".format(t))
        return True


Instruction:
Create a Python 3 function to
determine whether type t is a FHIR primitive type

Response:
def is_primitive(self, t: URIRef) -> bool:
        """
        Determine whether type "t" is a FHIR primitive type
        :param t: type to test
        :return:
        """
        return FHIR.Primitive in self._o.objects(t, RDFS.subClassOf)


Instruction:
Create a Python 3 function to
convert a predicate in the form of fhir : [.... value [ type ] to fhir : type

Response:
def value_predicate_to_type(self, value_pred: str) -> URIRef:
        """
        Convert a predicate in the form of "fhir:[...].value[type] to fhir:type, covering the downshift on the
        first character if necessary
        :param value_pred: Predicate associated with the value
        :return: corresponding type or None if not found
        """
        if value_pred.startswith('value'):
            vp_datatype = value_pred.replace('value', '')
            if vp_datatype:
                if self.has_type(FHIR[vp_datatype]):
                    return FHIR[vp_datatype]
                else:
                    vp_datatype = vp_datatype[0].lower() + vp_datatype[1:]
                    if self.has_type(FHIR[vp_datatype]):
                        return FHIR[vp_datatype]
        if self.is_valid(FHIR[value_pred]):
            return FHIR[value_pred]


Instruction:
Given the following Python 3 function, write the documentation
def is_atom(self, pred: URIRef) -> bool:
        if not self.has_type(pred):
            if '.value' in str(pred):               # synthetic values (valueString, valueDate, ...)
                return False
            else:
                raise TypeError("Unrecognized FHIR predicate: {}".format(pred))
        return pred == FHIR.nodeRole or OWL.DatatypeProperty in set(self._o.objects(pred, RDF.type))

Response: Determine whether the predicate is an atom type


Instruction:
Implement a function in Python 3 to
return the data type for primitive type t.

Response:
def primitive_datatype(self, t: URIRef) -> Optional[URIRef]:
        """
        Return the data type for primitive type t, if any
        :param t: type
        :return: corresponding data type
        """
        for sco in self._o.objects(t, RDFS.subClassOf):
            sco_type = self._o.value(sco, RDF.type)
            sco_prop = self._o.value(sco, OWL.onProperty)
            if sco_type == OWL.Restriction and sco_prop == FHIR.value:
                # The older versions of fhir.ttl (incorrectly) referenced the datatype directly
                restriction_type = self._o.value(sco, OWL.allValuesFrom)
                if not restriction_type:
                    restriction_dt_entry = self._o.value(sco, OWL.someValuesFrom)
                    restriction_type = self._o.value(restriction_dt_entry, OWL.onDatatype)
                return restriction_type
        return None


Instruction:
Write a Python 3 script to
return the data type for primitive type t if any defaulting string to no type

Response:
def primitive_datatype_nostring(self, t: URIRef, v: Optional[str] = None) -> Optional[URIRef]:
        """
        Return the data type for primitive type t, if any, defaulting string to no type
        :param t: type
        :param v: value - for munging dates if we're doing FHIR official output
        :return: corresponding data type
        """
        vt = self.primitive_datatype(t)
        if self.fhir_dates and vt == XSD.dateTime and v:
            return XSD.gYear if len(v) == 4 else XSD.gYearMonth if len(v) == 7 \
                else XSD.date if (len(v) == 10 or (len(v) > 10 and v[10] in '+-')) else XSD.dateTime
        # For some reason the oid datatype is represented as a string as well
        if self.fhir_oids and vt == XSD.anyURI:
            vt = None
        return None if vt == XSD.string else vt


Instruction:
Can you write a function in Python 3 where it
adds a file to the cache.

Response:
def add(self, name: str, sig: Tuple, obj: object) -> None:
        """
        Add a file to the cache
        :param name: name of the object to be pickled
        :param sig: signature for object
        :param obj: object to pickle
        """
        if self._cache_directory is not None:
            if name in self._cache:
                os.remove(os.path.join(self._cache_directory, self._cache[name].loc))
            fname = os.path.join(self._cache_directory, str(uuid.uuid4()))
            with open(fname, 'wb') as f:
                pickle.dump(obj, f)
            self._cache[name] = _PickleJar.CacheEntry(sig, fname)
            self._update()


Instruction:
How would you explain what the following Python 3 function does
def get(self, name: str, sig: Tuple) -> Optional[object]:
        if name not in self._cache:
            return None
        if self._cache[name].sig != sig:
            del self._cache[name]
            self._update()
            return None
        with open(self._cache[name].loc, 'rb') as f:
            return pickle.load(f)

Response: Returns the object representing name if it exists and has a unique signature.


Instruction:
Create a Python 3 function for
clearing all cache entries for the current instance.

Response:
def clear(self) -> None:
        """
        Clear all cache entries for directory and, if it is a 'pure' directory, remove the directory itself
        """
        if self._cache_directory is not None:
            # Safety - if there isn't a cache directory file, this probably isn't a valid cache
            assert os.path.exists(self._cache_directory_index), "Attempt to clear a non-existent cache"
            self._load()            # Shouldn't have any impact but...
            for e in self._cache.values():
                if os.path.exists(e.loc):
                    os.remove(e.loc)
            self._cache.clear()
            self._update()
        self._cache = {}