Instruction:
Implement a function in Python 3 to
check if function args don t match function signature

Response:
def check_function_args(args, signatures, function_name):
    """Check function args - return message if function args don't match function signature

    Called from validate_functions

    We have following types of arguments to validate:
        1. Required, position_dependent arguments, e.g. p(HGNC:AKT1), NSArg HGNC:AKT1 is required and must be first argument
        2. Optional, position_dependent arguments, e.g. pmod(P, T, 308) - T and 308 are optional and position_dependent
        3. Optional, e.g. loc() modifier can only be found once, but anywhere after the position_dependent arguments
        4. Multiple, e.g. var(), can have more than one var() modifier in p() function

    Args:
        args (Union['Function', 'NSArg', 'StrArg']): AST Function arguments
        signatures (Mapping[str, Any]): function signatures from spec_dict, may be more than one per function
        function_name (str): passed in to improve error messaging

    Returns:
        Tuple[bool, List[str]]: (function_valid?, list of error messages per signature)
    """

    messages = []

    arg_types = []
    for arg in args:
        arg_type = arg.__class__.__name__
        if arg_type == "Function":
            arg_types.append((arg.name, ""))
        elif arg_type == "NSArg":
            arg_types.append((arg_type, f"{arg.namespace}:{arg.value}"))
        elif arg_type == "StrArg":
            arg_types.append((arg_type, arg.value))
    log.debug(f"Arg_types {arg_types}")

    matched_signature_idx = -1
    valid_function = False
    for sig_argset_idx, sig_argset in enumerate(signatures):
        sig_req_args = sig_argset["req_args"]  # required position_dependent arguments
        sig_pos_args = sig_argset["pos_args"]  # optional position_dependent arguments
        sig_opt_args = sig_argset["opt_args"]  # optional arguments
        sig_mult_args = sig_argset["mult_args"]  # multiple arguments

        log.debug(f"{sig_argset_idx} Req: {sig_req_args}")
        log.debug(f"{sig_argset_idx} Pos: {sig_pos_args}")
        log.debug(f"{sig_argset_idx} Opt: {sig_opt_args}")
        log.debug(f"{sig_argset_idx} Mult: {sig_mult_args}")

        # Check required arguments
        reqs_mismatch_flag = False
        for sig_idx, sig_req in enumerate(sig_req_args):
            if len(arg_types) > sig_idx:
                log.debug(
                    "Req args: arg_type {} vs sig_req {}".format(
                        arg_types[sig_idx][0], sig_req
                    )
                )
                if arg_types[sig_idx][0] not in sig_req:
                    reqs_mismatch_flag = True
                    msg = f"Missing required arguments for {function_name} signature: {sig_argset_idx}"
                    messages.append(msg)
                    log.debug(msg)
                    break

        if reqs_mismatch_flag:
            continue  # test next argset

        # Check position_dependent optional arguments
        pos_dep_arg_types = arg_types[len(sig_req_args) :]
        log.debug(f"Optional arg types {pos_dep_arg_types}")
        log.debug(f"{sig_argset_idx} Pos: {sig_pos_args}")
        pos_mismatch_flag = False
        for sig_pos_idx, sig_pos in enumerate(sig_pos_args):
            if sig_pos_idx == len(pos_dep_arg_types):
                break  # stop checking position dependent arguments when we run out of them vs signature optional position dependent arguments
            if pos_dep_arg_types[sig_pos_idx][0] not in sig_pos:
                pos_mismatch_flag = True
                msg = f"Missing position_dependent arguments for {function_name} signature: {sig_argset_idx}"
                messages.append(msg)
                log.debug(msg)
                break
        if pos_mismatch_flag:
            continue  # test next argset

        reqpos_arglen = len(sig_req_args) + len(sig_pos_args)
        optional_arg_types = arg_types[reqpos_arglen:]

        # Remove function args that are found in the mult_args signature
        optional_types = [
            (opt_type, opt_val)
            for opt_type, opt_val in optional_arg_types
            if opt_type not in sig_mult_args
        ]
        log.debug(f"Optional types after sig mult args removed {optional_types}")

        # Check if any remaining function args are duplicated and therefore not unique opt_args
        if len(optional_types) != len(set(optional_types)):
            msg = f"Duplicate optional arguments {optional_types} for {function_name} signature: {sig_argset_idx}"
            messages.append(msg)
            log.debug(msg)
            continue

        optional_types = [
            (opt_type, opt_val)
            for opt_type, opt_val in optional_types
            if opt_type not in sig_opt_args
        ]
        if len(optional_types) > 0:
            msg = f"Invalid arguments {optional_types} for {function_name} signature: {sig_argset_idx}"
            messages.append(msg)
            log.debug(msg)
            continue

        matched_signature_idx = sig_argset_idx
        messages = []  # reset messages if signature is matched
        valid_function = True
        break

    # Add NSArg and StrArg value types (e.g. Protein, Complex, ec)
    if matched_signature_idx > -1:
        # Shouldn't have single optional NSArg arguments - not currently checking for that
        log.debug(
            f'AST1, Sigs {signatures[matched_signature_idx]["arguments"]}  Args: {args}'
        )
        for arg_idx, arg in enumerate(args):
            log.debug(f"Arg type {arg.type}")
            for sig_idx, sig_arg in enumerate(
                signatures[matched_signature_idx]["arguments"]
            ):
                if arg.type == "Function" or sig_arg["type"] in [
                    "Function",
                    "Modifier",
                ]:
                    pass  # Skip Function arguments
                elif sig_arg.get("position", None):
                    if sig_arg["position"] == arg_idx + 1:
                        arg.add_value_types(sig_arg["values"])
                        log.debug(f'AST2  {arg} {sig_arg["values"]}')
                elif arg.type in ["NSArg", "StrArg", "StrArgNSArg"]:
                    log.debug(f"Arg {arg}")
                    arg.add_value_types(sig_arg["values"])
                    log.debug(f'AST2  {arg} {sig_arg["values"]}')

    for arg in args:
        if arg.__class__.__name__ in ["NSArg", "StrArg"]:
            log.debug(f"Arg: {arg.to_string()} Value_types: {arg.value_types}")

    return (valid_function, messages)


Instruction:
Make a summary of the following Python 3 code
def validate_arg_values(ast, bo):

    if not bo.api_url:
        log.info("No API endpoint defined")
        return bo

    log.debug(f"AST: {ast}")

    # Test NSArg terms
    if isinstance(ast, NSArg):
        term_id = "{}:{}".format(ast.namespace, ast.value)
        value_types = ast.value_types
        log.debug(f"Value types: {value_types}  AST value: {ast.value}")
        # Default namespaces are defined in the bel_specification file
        if ast.namespace == "DEFAULT":  # may use the DEFAULT namespace or not
            for value_type in value_types:
                default_namespace = [
                    ns["name"] for ns in bo.spec["namespaces"][value_type]["info"]
                ] + [
                    ns["abbreviation"]
                    for ns in bo.spec["namespaces"][value_type]["info"]
                ]

                if ast.value in default_namespace:
                    log.debug("Default namespace valid term: {}".format(term_id))
                    break
            else:  # if for loop doesn't hit the break, run this else
                log.debug("Default namespace invalid term: {}".format(term_id))
                bo.validation_messages.append(
                    ("WARNING", f"Default Term: {term_id} not found")
                )

        # Process normal, non-default-namespace terms
        else:
            request_url = bo.api_url + "/terms/{}".format(
                url_path_param_quoting(term_id)
            )
            log.info(f"Validate Arg Values url {request_url}")
            r = get_url(request_url)
            if r and r.status_code == 200:
                result = r.json()
                # function signature term value_types doesn't match up with API term entity_types

                log.debug(
                    f'AST.value_types  {ast.value_types}  Entity types {result.get("entity_types", [])}'
                )

                # Check that entity types match
                if (
                    len(
                        set(ast.value_types).intersection(
                            result.get("entity_types", [])
                        )
                    )
                    == 0
                ):
                    log.debug(
                        "Invalid Term - statement term {} allowable entity types: {} do not match API term entity types: {}".format(
                            term_id, ast.value_types, result.get("entity_types", [])
                        )
                    )
                    bo.validation_messages.append(
                        (
                            "WARNING",
                            "Invalid Term - statement term {} allowable entity types: {} do not match API term entity types: {}".format(
                                term_id, ast.value_types, result.get("entity_types", [])
                            ),
                        )
                    )

                if term_id in result.get("obsolete_ids", []):
                    bo.validation_messages.append(
                        (
                            "WARNING",
                            f'Obsolete term: {term_id}  Current term: {result["id"]}',
                        )
                    )

            elif r.status_code == 404:
                bo.validation_messages.append(
                    ("WARNING", f"Term: {term_id} not found in namespace")
                )
            else:
                log.error(f"Status {r.status_code} - Bad URL: {request_url}")

    # Process StrArgs
    if isinstance(ast, StrArg):
        log.debug(f"  Check String Arg: {ast.value}  {ast.value_types}")
        for value_type in ast.value_types:
            # Is this a regex to match against
            if re.match("/", value_type):
                value_type = re.sub("^/", "", value_type)
                value_type = re.sub("/$", "", value_type)
                match = re.match(value_type, ast.value)
                if match:
                    break
            if value_type in bo.spec["namespaces"]:
                default_namespace = [
                    ns["name"] for ns in bo.spec["namespaces"][value_type]["info"]
                ] + [
                    ns["abbreviation"]
                    for ns in bo.spec["namespaces"][value_type]["info"]
                ]
                if ast.value in default_namespace:
                    break
        else:  # If for loop doesn't hit the break, no matches found, therefore for StrArg value is bad
            bo.validation_messages.append(
                (
                    "WARNING",
                    f"String value {ast.value} does not match default namespace value or regex pattern: {ast.value_types}",
                )
            )

    # Recursively process every NSArg by processing BELAst and Functions
    if hasattr(ast, "args"):
        for arg in ast.args:
            validate_arg_values(arg, bo)

    return bo

Response: Recursively validate NSArg and StrArg values


Instruction:
Can you create a Python 3 function that
gets BELBio configuration from files

Response:
def get_belbio_conf_files():
    """Get belbio configuration from files
    """

    home = os.path.expanduser("~")
    cwd = os.getcwd()

    belbio_conf_fp, belbio_secrets_fp = "", ""

    env_conf_dir = os.getenv("BELBIO_CONF", "").rstrip("/")

    conf_paths = [
        f"{cwd}/belbio_conf.yaml",
        f"{cwd}/belbio_conf.yml",
        f"{env_conf_dir}/belbio_conf.yaml",
        f"{env_conf_dir}/belbio_conf.yml",
        f"{home}/.belbio/conf",
    ]
    secret_paths = [
        f"{cwd}/belbio_secrets.yaml",
        f"{cwd}/belbio_secrets.yml",
        f"{env_conf_dir}/belbio_secrets.yaml",
        f"{env_conf_dir}/belbio_secrets.yml",
        f"{home}/.belbio/secrets",
    ]

    for fn in conf_paths:
        if os.path.exists(fn):
            belbio_conf_fp = fn
            break
    else:
        log.error(
            "No BELBio configuration file found - please add one (see http://bel.readthedocs.io/en/latest/configuration.html)"
        )

    for fn in secret_paths:
        if os.path.exists(fn):
            belbio_secrets_fp = fn
            break

    return (belbio_conf_fp, belbio_secrets_fp)


Instruction:
Implement a function in Python 3 to
load the configuration from the belbio_conf_fp belbio_secrets_fp

Response:
def load_configuration():
    """Load the configuration"""

    (belbio_conf_fp, belbio_secrets_fp) = get_belbio_conf_files()
    log.info(f"Using conf: {belbio_conf_fp} and secrets files: {belbio_secrets_fp} ")

    config = {}
    if belbio_conf_fp:
        with open(belbio_conf_fp, "r") as f:
            config = yaml.load(f, Loader=yaml.SafeLoader)
            config["source_files"] = {}
            config["source_files"]["conf"] = belbio_conf_fp

    if belbio_secrets_fp:
        with open(belbio_secrets_fp, "r") as f:
            secrets = yaml.load(f, Loader=yaml.SafeLoader)
            config["secrets"] = copy.deepcopy(secrets)
            if "source_files" in config:
                config["source_files"]["secrets"] = belbio_secrets_fp

    get_versions(config)

    # TODO - needs to be completed
    # add_environment_vars(config)

    return config


Instruction:
Explain what the following Python 3 code does
def get_versions(config) -> dict:

    # Collect bel package version
    try:
        import bel.__version__

        config["bel"]["version"] = bel.__version__.__version__
    except KeyError:
        config["bel"] = {"version": bel.__version__.__version__}
    except ModuleNotFoundError:
        pass

    # Collect bel_resources version
    try:
        import tools.__version__

        config["bel_resources"]["version"] = tools.__version__.__version__
    except KeyError:
        config["bel_resources"] = {"version": tools.__version__.__version__}
    except ModuleNotFoundError:
        pass

    # Collect bel_api version
    try:
        import __version__

        if __version__.__name__ == "BELBIO API":
            config["bel_api"]["version"] = __version__.__version__
    except KeyError:
        if __version__.__name__ == "BELBIO API":
            config["bel_api"] = {"version": __version__.__version__}
    except ModuleNotFoundError:
        pass

Response: Get versions of bel modules and tools


Instruction:
Can you create a Python 3 function that
overrides config with environment variables

Response:
def add_environment_vars(config: MutableMapping[str, Any]):
    """Override config with environment variables

    Environment variables have to be prefixed with BELBIO_
    which will be stripped before splitting on '__' and lower-casing
    the environment variable name that is left into keys for the
    config dictionary.

    Example:
        BELBIO_BEL_API__SERVERS__API_URL=http://api.bel.bio
        1. BELBIO_BEL_API__SERVERS__API_URL ==> BEL_API__SERVERS__API_URL
        2. BEL_API__SERVERS__API_URL ==> bel_api__servers__api_url
        3. bel_api__servers__api_url ==> [bel_api, servers, api_url]
        4. [bel_api, servers, api_url] ==> config['bel_api']['servers']['api_url'] = http://api.bel.bio

    """

    # TODO need to redo config - can't add value to dictionary without recursively building up the dict
    #         check into config libraries again

    for e in os.environ:
        if re.match("BELBIO_", e):
            val = os.environ.get(e)
            if val:
                e.replace("BELBIO_", "")
                env_keys = e.lower().split("__")
                if len(env_keys) > 1:
                    joined = '"]["'.join(env_keys)
                    eval_config = f'config["{joined}"] = val'
                    try:
                        eval(eval_config)
                    except Exception as exc:
                        log.warn("Cannot process {e} into config")
                else:
                    config[env_keys[0]] = val


Instruction:
Can you generate the documentation for the following Python 3 function
def merge_config(
    config: Mapping[str, Any],
    override_config: Mapping[str, Any] = None,
    override_config_fn: str = None,
) -> Mapping[str, Any]:
    """Override config with additional configuration in override_config or override_config_fn

    Used in script to merge CLI options with Config

    Args:
        config: original configuration
        override_config: new configuration to override/extend current config
        override_config_fn: new configuration filename as YAML file
    """

    if override_config_fn:
        with open(override_config_fn, "r") as f:
            override_config = yaml.load(f, Loader=yaml.SafeLoader)

    if not override_config:
        log.info("Missing override_config")

    return functools.reduce(rec_merge, (config, override_config))

Response:
Override config with additional configuration in override_config or override_config_fn
Shipment Used in script to merge CLI options with Config
Shipment


Instruction:
Can you generate the documentation for the following Python 3 function
def load_terms(fo: IO, metadata: dict, forceupdate: bool):

    version = metadata["metadata"]["version"]

    # LOAD TERMS INTO Elasticsearch
    with timy.Timer("Load Terms") as timer:
        es = bel.db.elasticsearch.get_client()

        es_version = version.replace("T", "").replace("-", "").replace(":", "")
        index_prefix = f"terms_{metadata['metadata']['namespace'].lower()}"
        index_name = f"{index_prefix}_{es_version}"

        # Create index with mapping
        if not elasticsearch.index_exists(es, index_name):
            elasticsearch.create_terms_index(es, index_name)
        elif forceupdate:  # force an update to the index
            index_name += "_alt"
            elasticsearch.create_terms_index(es, index_name)
        else:
            return  # Skip loading if not forced and not a new namespace

        terms_iterator = terms_iterator_for_elasticsearch(fo, index_name)
        elasticsearch.bulk_load_docs(es, terms_iterator)

        # Remove old namespace index
        index_names = elasticsearch.get_all_index_names(es)
        for name in index_names:
            if name != index_name and index_prefix in name:
                elasticsearch.delete_index(es, name)

        # Add terms_alias to this index
        elasticsearch.add_index_alias(es, index_name, terms_alias)

        log.info(
            "Load namespace terms",
            elapsed=timer.elapsed,
            namespace=metadata["metadata"]["namespace"],
        )

    # LOAD EQUIVALENCES INTO ArangoDB
    with timy.Timer("Load Term Equivalences") as timer:
        arango_client = arangodb.get_client()
        belns_db = arangodb.get_belns_handle(arango_client)
        arangodb.batch_load_docs(
            belns_db, terms_iterator_for_arangodb(fo, version), on_duplicate="update"
        )

        log.info(
            "Loaded namespace equivalences",
            elapsed=timer.elapsed,
            namespace=metadata["metadata"]["namespace"],
        )

        # Clean up old entries
        remove_old_equivalence_edges = f"""
            FOR edge in equivalence_edges
                FILTER edge.source == "{metadata["metadata"]["namespace"]}"
                FILTER edge.version != "{version}"
                REMOVE edge IN equivalence_edges
        """
        remove_old_equivalence_nodes = f"""
            FOR node in equivalence_nodes
                FILTER node.source == "{metadata["metadata"]["namespace"]}"
                FILTER node.version != "{version}"
                REMOVE node IN equivalence_nodes
        """
        arangodb.aql_query(belns_db, remove_old_equivalence_edges)
        arangodb.aql_query(belns_db, remove_old_equivalence_nodes)

    # Add metadata to resource metadata collection
    metadata["_key"] = f"Namespace_{metadata['metadata']['namespace']}"
    try:
        belns_db.collection(arangodb.belns_metadata_name).insert(metadata)
    except ArangoError as ae:
        belns_db.collection(arangodb.belns_metadata_name).replace(metadata)

Response: Load terms into Elasticsearch and ArangoDB


Instruction:
Explain what the following Python 3 code does
def terms_iterator_for_elasticsearch(fo: IO, index_name: str):

    species_list = config["bel_resources"].get("species_list", [])

    fo.seek(0)  # Seek back to beginning of file
    with gzip.open(fo, "rt") as f:
        for line in f:
            term = json.loads(line)
            # skip if not term record (e.g. is a metadata record)
            if "term" not in term:
                continue
            term = term["term"]

            # Filter species if enabled in config
            species_id = term.get("species_id", None)
            if species_list and species_id and species_id not in species_list:
                continue

            all_term_ids = set()
            for term_id in [term["id"]] + term.get("alt_ids", []):
                all_term_ids.add(term_id)
                all_term_ids.add(lowercase_term_id(term_id))

            term["alt_ids"] = copy.copy(list(all_term_ids))

            yield {
                "_op_type": "index",
                "_index": index_name,
                "_type": "term",
                "_id": term["id"],
                "_source": copy.deepcopy(term),
            }

Response: Iterate over the terms in the file that are in elastic search.


Instruction:
Can you tell what is the following Python 3 function doing
def lowercase_term_id(term_id: str) -> str:
    (ns, val) = term_id.split(":", maxsplit=1)
    term_id = f"{ns}:{val.lower()}"

    return term_id

Response: Lowercase the term value


Instruction:
Can you write a function in Python 3 where it
returns the text of a node.

Response:
def node_text(node):
    """Needed for things like abstracts which have internal tags (see PMID:27822475)"""

    if node.text:
        result = node.text
    else:
        result = ""
    for child in node:
        if child.tail is not None:
            result += child.tail
    return result


Instruction:
Explain what the following Python 3 code does
def get_pubtator(pmid):
    r = get_url(PUBTATOR_TMPL.replace("PMID", pmid), timeout=10)
    if r and r.status_code == 200:
        pubtator = r.json()[0]
    else:
        log.error(
            f"Cannot access Pubtator, status: {r.status_code} url: {PUBTATOR_TMPL.replace('PMID', pmid)}"
        )
        return None

    known_types = ["CHEBI", "Chemical", "Disease", "Gene", "Species"]

    for idx, anno in enumerate(pubtator["denotations"]):
        s_match = re.match(r"(\w+):(\w+)", anno["obj"])
        c_match = re.match(r"(\w+):(\w+):(\w+)", anno["obj"])
        if c_match:
            (ctype, namespace, cid) = (
                c_match.group(1),
                c_match.group(2),
                c_match.group(3),
            )

            if ctype not in known_types:
                log.info(f"{ctype} not in known_types for Pubtator")
            if namespace not in known_types:
                log.info(f"{namespace} not in known_types for Pubtator")

            pubtator["denotations"][idx][
                "obj"
            ] = f'{pubtator_ns_convert.get(namespace, "UNKNOWN")}:{cid}'
            pubtator["denotations"][idx]["entity_type"] = pubtator_entity_convert.get(
                ctype, None
            )
            pubtator["denotations"][idx][
                "annotation_type"
            ] = pubtator_annotation_convert.get(ctype, None)
        elif s_match:
            (ctype, cid) = (s_match.group(1), s_match.group(2))

            if ctype not in known_types:
                log.info(f"{ctype} not in known_types for Pubtator")

            pubtator["denotations"][idx][
                "obj"
            ] = f'{pubtator_ns_convert.get(ctype, "UNKNOWN")}:{cid}'
            pubtator["denotations"][idx]["entity_type"] = pubtator_entity_convert.get(
                ctype, None
            )
            pubtator["denotations"][idx][
                "annotation_type"
            ] = pubtator_annotation_convert.get(ctype, None)

    annotations = {}
    for anno in pubtator["denotations"]:
        log.info(anno)
        if anno["obj"] not in annotations:
            annotations[anno["obj"]] = {"spans": [anno["span"]]}
            annotations[anno["obj"]]["entity_types"] = [anno.get("entity_type", [])]
            annotations[anno["obj"]]["annotation_types"] = [
                anno.get("annotation_type", [])
            ]

        else:
            annotations[anno["obj"]]["spans"].append(anno["span"])

    del pubtator["denotations"]
    pubtator["annotations"] = copy.deepcopy(annotations)

    return pubtator

Response:
Get Pubtator Bioconcepts from Pubmed Abstract
 ArcGIS


Instruction:
Explain what the following Python 3 code does
def process_pub_date(year, mon, day):

    pub_date = None
    if year and re.match("[a-zA-Z]+", mon):
        pub_date = datetime.datetime.strptime(
            f"{year}-{mon}-{day}", "%Y-%b-%d"
        ).strftime("%Y-%m-%d")
    elif year:
        pub_date = f"{year}-{mon}-{day}"

    return pub_date

Response: Create pub_date from what Pubmed provides in Journal PubDate entry


Instruction:
Implement a function in Python 3 to
get pubmed xml for pmid and convert to JSON

Response:
def get_pubmed(pmid: str) -> Mapping[str, Any]:
    """Get pubmed xml for pmid and convert to JSON

    Remove MESH terms if they are duplicated in the compound term set

    ArticleDate vs PubDate gets complicated: https://www.nlm.nih.gov/bsd/licensee/elements_descriptions.html see <ArticleDate> and <PubDate>
    Only getting pub_year at this point from the <PubDate> element.

    Args:
        pmid: pubmed id number as a string

    Returns:
        pubmed json
    """
    pubmed_url = PUBMED_TMPL.replace("PMID", str(pmid))
    r = get_url(pubmed_url)
    log.info(f"Getting Pubmed URL {pubmed_url}")

    try:
        root = etree.fromstring(r.content)
        doc = {"abstract": ""}
        doc["pmid"] = root.xpath("//PMID/text()")[0]
        doc["title"] = next(iter(root.xpath("//ArticleTitle/text()")), "")

        # TODO https://stackoverflow.com/questions/4770191/lxml-etree-element-text-doesnt-return-the-entire-text-from-an-element
        atext = next(iter(root.xpath("//Abstract/AbstractText/text()")), "")
        print("Text", atext)

        for abstracttext in root.xpath("//Abstract/AbstractText"):
            abstext = node_text(abstracttext)

            label = abstracttext.get("Label", None)
            if label:
                doc["abstract"] += f"{label}: {abstext}\n"
            else:
                doc["abstract"] += f"{abstext}\n"

        doc["abstract"] = doc["abstract"].rstrip()

        doc["authors"] = []
        for author in root.xpath("//Author"):
            last_name = next(iter(author.xpath("LastName/text()")), "")
            first_name = next(iter(author.xpath("ForeName/text()")), "")
            initials = next(iter(author.xpath("Initials/text()")), "")
            if not first_name and initials:
                first_name = initials
            doc["authors"].append(f"{last_name}, {first_name}")

        pub_year = next(
            iter(root.xpath("//Journal/JournalIssue/PubDate/Year/text()")), None
        )
        pub_mon = next(
            iter(root.xpath("//Journal/JournalIssue/PubDate/Month/text()")), "Jan"
        )
        pub_day = next(
            iter(root.xpath("//Journal/JournalIssue/PubDate/Day/text()")), "01"
        )

        pub_date = process_pub_date(pub_year, pub_mon, pub_day)

        doc["pub_date"] = pub_date
        doc["journal_title"] = next(iter(root.xpath("//Journal/Title/text()")), "")
        doc["joural_iso_title"] = next(
            iter(root.xpath("//Journal/ISOAbbreviation/text()")), ""
        )
        doc["doi"] = next(iter(root.xpath('//ArticleId[@IdType="doi"]/text()')), None)

        doc["compounds"] = []
        for chem in root.xpath("//ChemicalList/Chemical/NameOfSubstance"):
            chem_id = chem.get("UI")
            doc["compounds"].append({"id": f"MESH:{chem_id}", "name": chem.text})

        compounds = [cmpd["id"] for cmpd in doc["compounds"]]
        doc["mesh"] = []
        for mesh in root.xpath("//MeshHeading/DescriptorName"):
            mesh_id = f"MESH:{mesh.get('UI')}"
            if mesh_id in compounds:
                continue
            doc["mesh"].append({"id": mesh_id, "name": mesh.text})

        return doc
    except Exception as e:
        log.error(
            f"Bad Pubmed request, status: {r.status_code} error: {e}",
            url=f'{PUBMED_TMPL.replace("PMID", pmid)}',
        )
        return {"message": f"Cannot get PMID: {pubmed_url}"}


Instruction:
How would you explain what the following Python 3 function does
def enhance_pubmed_annotations(pubmed: Mapping[str, Any]) -> Mapping[str, Any]:

    text = pubmed["title"] + pubmed["abstract"]

    annotations = {}

    for nsarg in pubmed["annotations"]:
        url = f'{config["bel_api"]["servers"]["api_url"]}/terms/{url_path_param_quoting(nsarg)}'
        log.info(f"URL: {url}")
        r = get_url(url)
        log.info(f"Result: {r}")
        new_nsarg = ""
        if r and r.status_code == 200:
            term = r.json()
            new_nsarg = bel_utils.convert_nsarg(term["id"], decanonicalize=True)

            pubmed["annotations"][nsarg]["name"] = term["name"]
            pubmed["annotations"][nsarg]["label"] = term["label"]
            pubmed["annotations"][nsarg]["entity_types"] = list(
                set(
                    pubmed["annotations"][nsarg]["entity_types"]
                    + term.get("entity_types", [])
                )
            )
            pubmed["annotations"][nsarg]["annotation_types"] = list(
                set(
                    pubmed["annotations"][nsarg]["annotation_types"]
                    + term.get("annotation_types", [])
                )
            )

        if new_nsarg != nsarg:
            annotations[new_nsarg] = copy.deepcopy(pubmed["annotations"][nsarg])
        else:
            annotations[nsarg] = copy.deepcopy(pubmed["annotations"][nsarg])

    for nsarg in annotations:
        for idx, span in enumerate(annotations[nsarg]["spans"]):
            string = text[span["begin"] - 1 : span["end"] - 1]
            annotations[nsarg]["spans"][idx]["text"] = string

    pubmed["annotations"] = copy.deepcopy(annotations)

    return pubmed

Response: Enhance pubmed annotations with the same namespace IDs


Instruction:
Explain what the following Python 3 code does
def get_pubmed_for_beleditor(pmid: str) -> Mapping[str, Any]:

    pubmed = get_pubmed(pmid)
    pubtator = get_pubtator(pmid)
    pubmed["annotations"] = copy.deepcopy(pubtator["annotations"])

    # Add entity types and annotation types to annotations
    pubmed = enhance_pubmed_annotations(pubmed)

    return pubmed

Response: Get the pubmed document with the given Pubmed PMID and add entity types and annotation types to the pubmed document.


Instruction:
Can you tell what is the following Python 3 function doing
def get_orthologs(canonical_gene_id: str, species: list = []) -> List[dict]:

    gene_id_key = bel.db.arangodb.arango_id_to_key(canonical_gene_id)
    orthologs = {}

    if species:
        query_filter = f"FILTER vertex.tax_id IN {species}"

    query = f"""
        LET start = (
            FOR vertex in ortholog_nodes
                FILTER vertex._key == "{gene_id_key}"
                RETURN {{ "name": vertex.name, "tax_id": vertex.tax_id }}
        )

        LET orthologs = (
            FOR vertex IN 1..3
                ANY "ortholog_nodes/{gene_id_key}" ortholog_edges
                OPTIONS {{ bfs: true, uniqueVertices : 'global' }}
                {query_filter}
                RETURN DISTINCT {{ "name": vertex.name, "tax_id": vertex.tax_id }}
        )

        RETURN {{ 'orthologs': FLATTEN(UNION(start, orthologs)) }}
    """

    cursor = belns_db.aql.execute(query, batch_size=20)
    results = cursor.pop()
    for ortholog in results["orthologs"]:
        norms = bel.terms.terms.get_normalized_terms(ortholog["name"])
        orthologs[ortholog["tax_id"]] = {
            "canonical": norms["canonical"],
            "decanonical": norms["decanonical"],
        }

    return orthologs

Response: Get orthologs for given gene_id and species


Instruction:
Write a Python 3 function that can
normalize strings with booleans into Python types.

Response:
def normalize_value(val):
    """
    Normalize strings with booleans into Python types.
    """

    if val is not None:
        if val.lower() == 'false':
            val = False
        elif val.lower() == 'true':
            val = True

    return val


Instruction:
Can you tell what is the following Python 3 function doing
def normalize_dictionary_values(dictionary):

    for key, val in dictionary.iteritems():
        if isinstance(val, dict):
            dictionary[key] = normalize_dictionary_values(val)
        elif isinstance(val, list):
            dictionary[key] = list(val)
        else:
            dictionary[key] = normalize_value(val)

    return dictionary

Response: Normalizes the values in a dictionary recursivly.


Instruction:
Can you generate the documentation for the following Python 3 function
def get_url(url: str, params: dict = {}, timeout: float = 5.0, cache: bool = True):

    try:

        if not cache:
            with requests_cache.disabled():
                r = requests.get(url, params=params, timeout=timeout)
        else:
            r = requests.get(url, params=params, timeout=timeout)

        log.debug(f"Response headers {r.headers}  From cache {r.from_cache}")
        return r

    except requests.exceptions.Timeout:
        log.warn(f"Timed out getting url in get_url: {url}")
        return None
    except Exception as e:
        log.warn(f"Error getting url: {url}  error: {e}")
        return None

Response: Wrapper for requests. get that handles timeout exceptions and returns the result object


Instruction:
Can you create a Python 3 function that
returns time in milliseconds from start_time

Response:
def timespan(start_time):
    """Return time in milliseconds from start_time"""

    timespan = datetime.datetime.now() - start_time
    timespan_ms = timespan.total_seconds() * 1000
    return timespan_ms


Instruction:
Can you generate the documentation for the following Python 3 function
def download_file(url):

    response = requests.get(url, stream=True)
    fp = tempfile.NamedTemporaryFile()
    for chunk in response.iter_content(chunk_size=1024):
        if chunk:  # filter out keep-alive new chunks
            fp.write(chunk)

    # log.info(f'Download file - tmp file: {fp.name}  size: {fp.tell()}')
    return fp

Response: Download file from url and return file object


Instruction:
How would you implement a function in Python 3 that
returns the first true value in the iterable.

Response:
def first_true(iterable, default=False, pred=None):
    """Returns the first true value in the iterable.

    If no true value is found, returns *default*

    If *pred* is not None, returns the first item
    for which pred(item) is true.

    """
    # first_true([a,b,c], x) --> a or b or c or x
    # first_true([a,b], x, f) --> a if f(a) else b if f(b) else x
    return next(filter(pred, iterable), default)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _create_hash_from_doc(doc: Mapping[str, Any]) -> str:

    doc_string = json.dumps(doc, sort_keys=True)
    return _create_hash(doc_string)

Response: Create hash Id from document


Instruction:
Given the following Python 3 function, write the documentation
def elapsed(self):
        if self.end is None:
            # if elapsed is called in the context manager scope
            return (self() - self.start) * self.factor
        else:
            # if elapsed is called out of the context manager scope
            return (self.end - self.start) * self.factor

Response: Return the elapsed time since the start of the current block.


Instruction:
Can you implement a function in Python 3 that
loads edges into Arangodb

Response:
def load_edges_into_db(
    nanopub_id: str,
    nanopub_url: str,
    edges: list = [],
    edges_coll_name: str = edges_coll_name,
    nodes_coll_name: str = nodes_coll_name,
):
    """Load edges into Edgestore"""

    start_time = datetime.datetime.now()

    # Clean out edges for nanopub in edgestore
    query = f"""
        FOR edge IN {edges_coll_name}
            FILTER edge.nanopub_id == "{nanopub_id}"
            REMOVE edge IN edges
    """

    try:
        edgestore_db.aql.execute(query)
    except Exception as e:
        log.debug(f"Could not remove nanopub-related edges: {query}  msg: {e}")

    end_time1 = datetime.datetime.now()
    delta_ms = f"{(end_time1 - start_time).total_seconds() * 1000:.1f}"
    log.info("Timing - Delete edges for nanopub", delta_ms=delta_ms)

    # Clean out errors for nanopub in pipeline_errors
    query = f"""
        FOR e IN pipeline_errors
            FILTER e.nanopub_id == "{nanopub_id}"
            REMOVE e IN pipeline_errors
    """
    try:
        edgestore_db.aql.execute(query)
    except Exception as e:
        log.debug(f"Could not remove nanopub-related errors: {query} msg: {e}")

    end_time2 = datetime.datetime.now()
    delta_ms = f"{(end_time2 - end_time1).total_seconds() * 1000:.1f}"
    log.info("Timing - Delete pipeline errors for nanopub", delta_ms=delta_ms)

    # Collect edges and nodes to load into arangodb
    node_list, edge_list = [], []
    for doc in edge_iterator(edges=edges):
        if doc[0] == "nodes":
            node_list.append(doc[1])
        else:
            edge_list.append(doc[1])

    end_time3 = datetime.datetime.now()
    delta_ms = f"{(end_time3 - end_time2).total_seconds() * 1000:.1f}"
    log.info("Timing - Collect edges and nodes", delta_ms=delta_ms)

    try:
        results = edgestore_db.collection(edges_coll_name).import_bulk(
            edge_list, on_duplicate="replace", halt_on_error=False
        )
    except Exception as e:
        log.error(f"Could not load edges  msg: {e}")

    end_time4 = datetime.datetime.now()
    delta_ms = f"{(end_time4 - end_time3).total_seconds() * 1000:.1f}"
    log.info("Timing - Load edges into edgestore", delta_ms=delta_ms)

    try:
        results = edgestore_db.collection(nodes_coll_name).import_bulk(
            node_list, on_duplicate="replace", halt_on_error=False
        )
    except Exception as e:
        log.error(f"Could not load nodes  msg: {e}")

    end_time5 = datetime.datetime.now()
    delta_ms = f"{(end_time5 - end_time4).total_seconds() * 1000:.1f}"
    log.info("Timing - Load nodes into edgestore", delta_ms=delta_ms)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def edge_iterator(edges=[], edges_fn=None):

    for edge in itertools.chain(edges, files.read_edges(edges_fn)):

        subj = copy.deepcopy(edge["edge"]["subject"])
        subj_id = str(utils._create_hash_from_doc(subj))
        subj["_key"] = subj_id
        obj = copy.deepcopy(edge["edge"]["object"])

        obj_id = str(utils._create_hash_from_doc(obj))
        obj["_key"] = obj_id
        relation = copy.deepcopy(edge["edge"]["relation"])

        relation["_from"] = f"nodes/{subj_id}"
        relation["_to"] = f"nodes/{obj_id}"

        # Create edge _key
        relation_hash = copy.deepcopy(relation)
        relation_hash.pop("edge_dt", None)
        relation_hash.pop("edge_hash", None)
        relation_hash.pop("nanopub_dt", None)
        relation_hash.pop("nanopub_url", None)
        relation_hash.pop("subject_canon", None)
        relation_hash.pop("object_canon", None)
        relation_hash.pop("public_flag", None)
        relation_hash.pop("metadata", None)

        relation_id = str(utils._create_hash_from_doc(relation_hash))
        relation["_key"] = relation_id

        if edge.get("nanopub_id", None):
            if "metadata" not in relation:
                relation["metadata"] = {}
            relation["metadata"]["nanopub_id"] = edge["nanopub_id"]

        yield ("nodes", subj)
        yield ("nodes", obj)
        yield ("edges", relation)

Response: Yields all documents from an edge for loading into ArangoDB


Instruction:
Implement a Python 3 function for
updating nanopubstore start_dt in belapi. state_mgmt collection

Response:
def update_nanopubstore_start_dt(url: str, start_dt: str):
    """Add nanopubstore start_dt to belapi.state_mgmt collection

    Args:
        url: url of nanopubstore
        start_dt: datetime of last query against nanopubstore for new ID's
    """

    hostname = urllib.parse.urlsplit(url)[1]

    start_dates_doc = state_mgmt.get(start_dates_doc_key)
    if not start_dates_doc:
        start_dates_doc = {
            "_key": start_dates_doc_key,
            "start_dates": [{"nanopubstore": hostname, "start_dt": start_dt}],
        }
        state_mgmt.insert(start_dates_doc)
    else:
        for idx, start_date in enumerate(start_dates_doc["start_dates"]):
            if start_date["nanopubstore"] == hostname:
                start_dates_doc["start_dates"][idx]["start_dt"] = start_dt
                break
        else:
            start_dates_doc["start_dates"].append(
                {"nanopubstore": hostname, "start_dt": start_dt}
            )

        state_mgmt.replace(start_dates_doc)


Instruction:
How would you code a function in Python 3 to
get last start_dt recorded for getting new nanopub ID s

Response:
def get_nanopubstore_start_dt(url: str):
    """Get last start_dt recorded for getting new nanopub ID's"""

    hostname = urllib.parse.urlsplit(url)[1]

    start_dates_doc = state_mgmt.get(start_dates_doc_key)
    if start_dates_doc and start_dates_doc.get("start_dates"):
        date = [
            dt["start_dt"]
            for dt in start_dates_doc["start_dates"]
            if dt["nanopubstore"] == hostname
        ]
        log.info(f"Selected start_dt: {date}  len: {len(date)}")
        if len(date) == 1:
            return date[0]

    return "1900-01-01T00:00:00.000Z"


Instruction:
How would you code a function in Python 3 to
get modified and deleted nanopub urls

Response:
def get_nanopub_urls(ns_root_url: str = None, start_dt: str = None) -> dict:
    """Get modified and deleted nanopub urls

    Limited by last datetime retrieved (start_dt).  Modified includes new and updated nanopubs

    Returns:
        dict: {'modified': [], 'deleted': []}
    """
    if not ns_root_url:
        ns_root_url = config["bel_api"]["servers"]["nanopubstore"]

    url = f"{ns_root_url}/nanopubs/timed"

    if not start_dt:
        start_dt = get_nanopubstore_start_dt(ns_root_url)

    params = {"startTime": start_dt, "published": True}

    # TODO - this is coming back without a status code in some cases - why?
    r = bel.utils.get_url(url, params=params, cache=False)
    if r and r.status_code == 200:
        data = r.json()
        new_start_dt = data["queryTime"]
        update_nanopubstore_start_dt(ns_root_url, new_start_dt)
        nanopub_urls = {"modified": [], "deleted": []}

        # Deleted nanopubs
        for nid in data["deleteddata"]:
            nanopub_urls["deleted"].append(f"{ns_root_url}/nanopubs/{nid}")

        # Modified nanopubs
        for nid in data["data"]:
            nanopub_urls["modified"].append(f"{ns_root_url}/nanopubs/{nid}")

        return nanopub_urls

    else:
        log.error(
            f"Bad request to Nanopubstore",
            url=url,
            status=r.status_code,
            type="api_request",
        )
        return {}


Instruction:
Can you generate the documentation for the following Python 3 function
def get_nanopub(url):

    r = bel.utils.get_url(url, cache=False)
    if r and r.json():
        return r.json()
    else:
        return {}

Response: Get Nanopub from nanopubstore given url


Instruction:
Can you generate a brief explanation for the following Python 3 code
def pipeline(
    ctx,
    input_fn,
    db_save,
    db_delete,
    output_fn,
    rules,
    species,
    namespace_targets,
    version,
    api,
    config_fn,
):
    """BEL Pipeline - BEL Nanopubs into BEL Edges

    This will process BEL Nanopubs into BEL Edges by validating, orthologizing (if requested),
    canonicalizing, and then computing the BEL Edges based on the given rule_set.

    \b
    input_fn:
        If input fn has *.gz, will read as a gzip file
        If input fn has *.jsonl*, will parsed as a JSONLines file
        IF input fn has *.json*, will be parsed as a JSON file
        If input fn has *.yaml* or *.yml*,  will be parsed as a YAML file

    \b
    output_fn:
        If output fn has *.gz, will written as a gzip file
        If output fn has *.jsonl*, will written as a JSONLines file
        IF output fn has *.json*, will be written as a JSON file
        If output fn has *.yaml* or *.yml*,  will be written as a YAML file
        If output fn has *.jgf, will be written as JSON Graph Formatted file
    """

    if config_fn:
        config = bel.db.Config.merge_config(ctx.config, override_config_fn=config_fn)
    else:
        config = ctx.config

    # Configuration - will return the first truthy result in list else the default option
    if namespace_targets:
        namespace_targets = json.loads(namespace_targets)
    if rules:
        rules = rules.replace(" ", "").split(",")

    namespace_targets = utils.first_true(
        [namespace_targets, config["bel"]["lang"].get("canonical")], None
    )
    rules = utils.first_true(
        [rules, config["bel"]["nanopub"].get("pipeline_edge_rules", False)], False
    )
    api = utils.first_true(
        [api, config["bel_api"]["servers"].get("api_url", None)], None
    )
    version = utils.first_true(
        [version, config["bel"]["lang"].get("default_bel_version", None)], None
    )

    n = bnn.Nanopub()

    try:
        json_flag, jsonl_flag, yaml_flag, jgf_flag = False, False, False, False
        all_bel_edges = []
        fout = None

        if db_save or db_delete:
            if db_delete:
                arango_client = bel.db.arangodb.get_client()
                bel.db.arangodb.delete_database(arango_client, "edgestore")
            else:
                arango_client = bel.db.arangodb.get_client()

            edgestore_handle = bel.db.arangodb.get_edgestore_handle(arango_client)

        elif re.search("ya?ml", output_fn):
            yaml_flag = True
        elif "jsonl" in output_fn:
            jsonl_flag = True
        elif "json" in output_fn:
            json_flag = True
        elif "jgf" in output_fn:
            jgf_flag = True

        if db_save:
            pass
        elif "gz" in output_fn:
            fout = gzip.open(output_fn, "wt")
        else:
            fout = open(output_fn, "wt")

        nanopub_cnt = 0
        with timy.Timer() as timer:
            for np in bnf.read_nanopubs(input_fn):
                # print('Nanopub:\n', json.dumps(np, indent=4))

                nanopub_cnt += 1
                if nanopub_cnt % 100 == 0:
                    timer.track(f"{nanopub_cnt} Nanopubs processed into Edges")

                bel_edges = n.bel_edges(
                    np,
                    namespace_targets=namespace_targets,
                    orthologize_target=species,
                    rules=rules,
                )

                if db_save:
                    bel.edge.edges.load_edges_into_db(edgestore_handle, edges=bel_edges)
                elif jsonl_flag:
                    fout.write("{}\n".format(json.dumps(bel_edges)))
                else:
                    all_bel_edges.extend(bel_edges)

        if db_save:
            pass
        elif yaml_flag:
            fout.write("{}\n".format(yaml.dumps(all_bel_edges)))
        elif json_flag:
            fout.write("{}\n".format(json.dumps(all_bel_edges)))
        elif jgf_flag:
            bnf.edges_to_jgf(output_fn, all_bel_edges)

    finally:
        if fout:
            fout.close()

Response: This function processes the BEL Nanopubs into BEL Edges and returns the result as a list of BEL Nanopubs.


Instruction:
Make a summary of the following Python 3 code
def convert_belscript(ctx, input_fn, output_fn):

    try:

        (
            out_fh,
            yaml_flag,
            jsonl_flag,
            json_flag,
        ) = bel.nanopub.files.create_nanopubs_fh(output_fn)
        if yaml_flag or json_flag:
            docs = []

        # input file
        if re.search("gz$", input_fn):
            f = gzip.open(input_fn, "rt")
        else:
            f = open(input_fn, "rt")

        # process belscript
        for doc in bel.nanopub.belscripts.parse_belscript(f):
            if yaml_flag or json_flag:
                docs.append(doc)
            elif jsonl_flag:
                out_fh.write("{}\n".format(json.dumps(doc)))

        if yaml_flag:
            yaml.dump(docs, out_fh)

        elif json_flag:
            json.dump(docs, out_fh, indent=4)

    finally:
        f.close()
        out_fh.close()

Response: Convert BEL script to nanopubs_bel format


Instruction:
How would you implement a function in Python 3 that
reformats between JSON YAML and JSONLines files.

Response:
def reformat(ctx, input_fn, output_fn):
    """Reformat between JSON, YAML, JSONLines formats

    \b
    input_fn:
        If input fn has *.gz, will read as a gzip file

    \b
    output_fn:
        If output fn has *.gz, will written as a gzip file
        If output fn has *.jsonl*, will written as a JSONLines file
        IF output fn has *.json*, will be written as a JSON file
        If output fn has *.yaml* or *.yml*,  will be written as a YAML file
    """

    try:

        (
            out_fh,
            yaml_flag,
            jsonl_flag,
            json_flag,
        ) = bel.nanopub.files.create_nanopubs_fh(output_fn)
        if yaml_flag or json_flag:
            docs = []

        # input file
        if re.search("gz$", input_fn):
            f = gzip.open(input_fn, "rt")
        else:
            f = open(input_fn, "rt")

        for np in bnf.read_nanopubs(input_fn):
            if yaml_flag or json_flag:
                docs.append(np)
            elif jsonl_flag:
                out_fh.write("{}\n".format(json.dumps(np)))

        if yaml_flag:
            yaml.dump(docs, out_fh)

        elif json_flag:
            json.dump(docs, out_fh, indent=4)

    finally:
        f.close()
        out_fh.close()


Instruction:
Create a Python 3 function to
collect statistics on nanopub file input_fn can be json jsonl or yaml and additionally gzipped

Response:
def nanopub_stats(ctx, input_fn):
    """Collect statistics on nanopub file

    input_fn can be json, jsonl or yaml and additionally gzipped
    """

    counts = {
        "nanopubs": 0,
        "assertions": {"total": 0, "subject_only": 0, "nested": 0, "relations": {}},
    }

    for np in bnf.read_nanopubs(input_fn):
        if "nanopub" in np:
            counts["nanopubs"] += 1
            counts["assertions"]["total"] += len(np["nanopub"]["assertions"])
            for assertion in np["nanopub"]["assertions"]:
                if assertion["relation"] is None:
                    counts["assertions"]["subject_only"] += 1
                else:
                    if re.match("\s*\(", assertion["object"]):
                        counts["assertions"]["nested"] += 1

                    if (
                        not assertion.get("relation")
                        in counts["assertions"]["relations"]
                    ):
                        counts["assertions"]["relations"][assertion.get("relation")] = 1
                    else:
                        counts["assertions"]["relations"][
                            assertion.get("relation")
                        ] += 1

    counts["assertions"]["relations"] = sorted(counts["assertions"]["relations"])

    print("DumpVar:\n", json.dumps(counts, indent=4))


Instruction:
Explain what the following Python 3 code does
def edges(ctx, statement, rules, species, namespace_targets, version, api, config_fn):

    if config_fn:
        config = bel.db.Config.merge_config(ctx.config, override_config_fn=config_fn)
    else:
        config = ctx.config

    # Configuration - will return the first truthy result in list else the default option
    if namespace_targets:
        namespace_targets = json.loads(namespace_targets)
    if rules:
        rules = rules.replace(" ", "").split(",")

    namespace_targets = utils.first_true(
        [namespace_targets, config["bel"]["lang"].get("canonical")], None
    )
    api_url = utils.first_true(
        [api, config["bel_api"]["servers"].get("api_url", None)], None
    )
    version = utils.first_true(
        [version, config["bel"]["lang"].get("default_bel_version", None)], None
    )

    print("------------------------------")
    print("BEL version: {}".format(version))
    print("API Endpoint: {}".format(api))
    print("------------------------------")

    bo = BEL(version=version, endpoint=api_url)
    if species:
        edges = (
            bo.parse(statement)
            .orthologize(species)
            .canonicalize(namespace_targets=namespace_targets)
            .compute_edges(rules=rules)
        )
    else:
        edges = (
            bo.parse(statement)
            .canonicalize(namespace_targets=namespace_targets)
            .compute_edges(rules=rules)
        )

    if edges is None:
        print(bo.original_bel_stmt)
        print(bo.parse_visualize_error)
        print(bo.validation_messages)
    else:
        print(json.dumps(edges, indent=4))

        if bo.validation_messages:
            print(bo.validation_messages)
        else:
            print("No problems found")
    return

Response: Create BEL Edges from BEL Statement


Instruction:
Given the following Python 3 function, write the documentation
def validate_to_schema(nanopub, schema) -> Tuple[bool, List[Tuple[str, str]]]:

    v = jsonschema.Draft4Validator(schema)
    messages = []
    errors = sorted(v.iter_errors(nanopub), key=lambda e: e.path)
    for error in errors:
        for suberror in sorted(error.context, key=lambda e: e.schema_path):
            print(list(suberror.schema_path), suberror.message, sep=", ")
            messages.append(("ERROR", suberror.message))

    is_valid = True
    if errors:
        is_valid = False

    return (is_valid, messages)

Response:
Validate nanopub against jsonschema for nanopub
 ArcGIS


Instruction:
Can you create a Python 3 function that
creates CityHash64 from nanopub

Response:
def hash_nanopub(nanopub: Mapping[str, Any]) -> str:
    """Create CityHash64 from nanopub for duplicate check

    TODO - check that this hash value is consistent between C# and Python running on
    laptop and server

    Build string to hash

    Collect flat array of (all values.strip()):
        nanopub.type.name
        nanopub.type.version

        One of:
            nanopub.citation.database.name
            nanopub.citation.database.id

            OR

            nanopub.citation.database.uri

            OR

            nanopub.citation.database.reference

        Extend with sorted list of assertions (SRO as single string with space between S, R and O)

        Extend with sorted list of annotations (nanopub.annotations.type + ' ' + nanopub.annotations.id)

    Convert array to string by joining array elements separated by a space

    Create CityHash64(str) and return

    """

    hash_list = []

    # Type
    hash_list.append(nanopub["nanopub"]["type"].get("name", "").strip())
    hash_list.append(nanopub["nanopub"]["type"].get("version", "").strip())

    # Citation
    if nanopub["nanopub"]["citation"].get("database", False):
        hash_list.append(
            nanopub["nanopub"]["citation"]["database"].get("name", "").strip()
        )
        hash_list.append(
            nanopub["nanopub"]["citation"]["database"].get("id", "").strip()
        )
    elif nanopub["nanopub"]["citation"].get("uri", False):
        hash_list.append(nanopub["nanopub"]["citation"].get("uri", "").strip())
    elif nanopub["nanopub"]["citation"].get("reference", False):
        hash_list.append(nanopub["nanopub"]["citation"].get("reference", "").strip())

    # Assertions
    assertions = []
    for assertion in nanopub["nanopub"]["assertions"]:
        if assertion.get("relation") is None:
            assertion["relation"] = ""
        if assertion.get("object") is None:
            assertion["object"] = ""
        assertions.append(
            " ".join(
                (
                    assertion["subject"].strip(),
                    assertion.get("relation", "").strip(),
                    assertion.get("object", "").strip(),
                )
            ).strip()
        )
    assertions = sorted(assertions)
    hash_list.extend(assertions)

    # Annotations
    annotations = []

    for anno in nanopub["nanopub"]["annotations"]:
        annotations.append(
            " ".join((anno.get("type", "").strip(), anno.get("id", "").strip())).strip()
        )

    annotations = sorted(annotations)
    hash_list.extend(annotations)

    np_string = " ".join([l.lower() for l in hash_list])

    return "{:x}".format(CityHash64(np_string))


Instruction:
Can you create a Python 3 function that
validates the nanopub dict using the nanopub schema.

Response:
def validate(
        self, nanopub: Mapping[str, Any]
    ) -> Tuple[bool, List[Tuple[str, str]]]:
        """Validates using the nanopub schema

        Args:
            nanopub (Mapping[str, Any]): nanopub dict

        Returns:
            Tuple[bool, List[Tuple[str, str]]]:
                bool: Is valid?  Yes = True, No = False
                List[Tuple[str, str]]: Validation issues, empty if valid, tuple is ('ERROR|WARNING', msg)
                    e.g. [('WARNING', "Context ID not found")]        """

        # Validate nanopub
        (is_valid, messages) = validate_to_schema(nanopub, self.nanopub_schema)
        if not is_valid:
            return messages

        # Extract BEL Version
        if nanopub["nanopub"]["type"]["name"].upper() == "BEL":
            bel_version = nanopub["nanopub"]["type"]["version"]
        else:
            is_valid = False
            return (
                is_valid,
                f"Not a BEL Nanopub according to nanopub.type.name: {nanopub['nanopub']['type']['name']}",
            )

        all_messages = []
        # Validate BEL Statements
        bel_obj = bel.lang.belobj.BEL(bel_version, self.endpoint)
        for edge in nanopub["nanopub"]["edges"]:
            bel_statement = f"{edge['subject']} {edge['relation']} {edge['object']}"
            parse_obj = bel_obj.parse(bel_statement)
            if not parse_obj.valid:
                all_messages.extend(
                    (
                        "ERROR",
                        f"BEL statement parse error {parse_obj.error}, {parse_obj.err_visual}",
                    )
                )

        # Validate nanopub.context
        for context in nanopub["nanopub"]["context"]:
            (is_valid, messages) = self.validate_context(context)
            all_messages.extend(messages)

        is_valid = True
        for _type, msg in all_messages:
            if _type == "ERROR":
                is_valid = False

        return (is_valid, all_messages)


Instruction:
Write a Python 3 script to
validate a context dictionary of type id and label.

Response:
def validate_context(
        self, context: Mapping[str, Any]
    ) -> Tuple[bool, List[Tuple[str, str]]]:
        """ Validate context

        Args:
            context (Mapping[str, Any]): context dictionary of type, id and label

        Returns:
            Tuple[bool, List[Tuple[str, str]]]:
                bool: Is valid?  Yes = True, No = False
                List[Tuple[str, str]]: Validation issues, empty if valid, tuple is ('ERROR|WARNING', msg)
                    e.g. [('WARNING', "Context ID not found")]
        """

        url = f'{self.endpoint}/terms/{context["id"]}'

        res = requests.get(url)
        if res.status_code == 200:
            return (True, [])
        else:
            return (False, [("WARNING", f'Context {context["id"]} not found at {url}')])


Instruction:
Implement a Python 3 function for
creating BEL Edges from BEL nanopub.

Response:
def bel_edges(
        self,
        nanopub: Mapping[str, Any],
        namespace_targets: Mapping[str, List[str]] = {},
        rules: List[str] = [],
        orthologize_target: str = None,
    ) -> List[Mapping[str, Any]]:
        """Create BEL Edges from BEL nanopub

        Args:
            nanopub (Mapping[str, Any]): bel nanopub
            namespace_targets (Mapping[str, List[str]]): what namespaces to canonicalize
            rules (List[str]): which computed edge rules to process, default is all,
               look at BEL Specification yaml file for computed edge signature keys,
               e.g. degradation, if any rule in list is 'skip', then skip computing edges
               just return primary_edge
            orthologize_target (str): species to convert BEL into, e.g. TAX:10090 for mouse, default option does not orthologize

        Returns:
            List[Mapping[str, Any]]: edge list with edge attributes (e.g. context)
        """

        edges = bel.edge.edges.create_edges(
            nanopub,
            self.endpoint,
            namespace_targets=namespace_targets,
            rules=rules,
            orthologize_target=orthologize_target,
        )

        return edges


Instruction:
Here you have a function in Python 3, explain what it does
def main_hrun():

    parser = argparse.ArgumentParser(description="Tools for http(s) test. Base on rtsf.")

    parser.add_argument(
        '--log-level', default='INFO',
        help="Specify logging level, default is INFO.")

    parser.add_argument(
        '--log-file',
        help="Write logs to specified file path.")

    parser.add_argument(
        'case_file',
        help="yaml testcase file")

    color_print("httpdriver {}".format(__version__), "GREEN")
    args = parser.parse_args()
    logger.setup_logger(args.log_level, args.log_file)

    runner = TestRunner(runner = HttpDriver).run(args.case_file)
    html_report = runner.gen_html_report()
    color_print("report: {}".format(html_report))

Response: parse command line options and run commands


Instruction:
How would you implement a function in Python 3 that
finds the. pyd file in the build directory.

Response:
def find_pyd_file():
    """
    Return path to .pyd after successful build command.
    :return: Path to .pyd file or None.
    """

    if not os.path.isdir("./build"):
        raise NotADirectoryError

    for path, dirs, files in os.walk("./build"):
        for file_name in files:
            file_name_parts = os.path.splitext(file_name)
            if file_name_parts[1] == ".pyd":
                return path
    return None


Instruction:
Can you tell what is the following Python 3 function doing
def _void_array_to_nested_list(res, _func, _args):
    try:
        shape = res.coords.len, 2
        ptr = cast(res.coords.data, POINTER(c_double))
        array = np.ctypeslib.as_array(ptr, shape)
        return array.tolist()
    finally:
        drop_array(res.coords)

Response: Dereference the FFI result to a list of coordinates


Instruction:
How would you explain what the following Python 3 function does
def is_json_file(abspath):
    abspath = abspath.lower()
    fname, ext = os.path.splitext(abspath)
    if ext in [".json", ".js"]:
        is_json = True
    elif ext == ".gz":
        is_json = False
    elif ext == ".tmp":
        return is_json_file(fname)
    else:
        raise JsonExtError(
            "'%s' is not a valid json file. "
            "extension has to be '.json' for uncompressed, '.gz' "
            "for compressed." % abspath)
    return is_json

Response: Check if file is a json file.


Instruction:
Can you tell what is the following Python 3 function doing
def lower_ext(abspath):
    fname, ext = os.path.splitext(abspath)
    return fname + ext.lower()

Response: Convert file extension to lowercase.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def load(abspath, default=None, enable_verbose=True):
    if default is None:
        default = dict()

    prt("\nLoad from '%s' ..." % abspath, enable_verbose)

    abspath = lower_ext(str(abspath))
    is_json = is_json_file(abspath)

    if not os.path.exists(abspath):
        prt("    File not found, use default value: %r" %
            default, enable_verbose)
        return default

    st = time.clock()
    if is_json:
        data = json.loads(textfile.read(abspath, encoding="utf-8"))
    else:
        data = json.loads(compress.read_gzip(abspath).decode("utf-8"))

    prt("    Complete! Elapse %.6f sec." % (time.clock() - st), enable_verbose)
    return data

Response: Load a json file into a dict.


Instruction:
Write a Python 3 function that can
dump a JSON serializable object to a file.

Response:
def dump(data, abspath,
         indent_format=False,
         float_precision=None,
         ensure_ascii=True,
         overwrite=False,
         enable_verbose=True):
    """Dump Json serializable object to file.
    Provides multiple choice to customize the behavior.

    :param data: Serializable python object.
    :type data: dict or list

    :param abspath: ``save as`` path, file extension has to be ``.json`` or ``.gz``
        (for compressed Json)
    :type abspath: string

    :param indent_format: default ``False``, If ``True``, then dump to human
      readable format, but it's slower, the file is larger
    :type indent_format: boolean

    :param float_precision: default ``None``, limit flotas to N-decimal points.
    :type float_precision: integer

    :param overwrite: default ``False``, If ``True``, when you dump to existing
        file, it silently overwrite it. If ``False``, an alert message is shown.
        Default setting ``False`` is to prevent overwrite file by mistake.
    :type overwrite: boolean

    :param enable_verbose: default True, help-message-display trigger.
    :type enable_verbose: boolean

    Usage::

        >>> from dataIO import js
        >>> data = {"a": 1, "b": 2}
        >>> dump(data, "test.json", overwrite=True)
        Dumping to 'test.json'...
            Complete! Elapse 0.002432 sec

    **中文文档**

    将Python中可被序列化的"字典", "列表"以及他们的组合, 按照Json的编码方式写入文件
    文件

    参数列表

    :param js: 可Json化的Python对象
    :type js: ``字典`` 或 ``列表``

    :param abspath: Json文件绝对路径, 扩展名需为 ``.json`` 或 ``.gz``, 其中 ``.gz``
      是被压缩后的Json文件
    :type abspath: ``字符串``

    :param indent_format: 默认 ``False``, 当为 ``True`` 时, Json编码时会对Key进行
      排序, 并进行缩进排版。但是这样写入速度较慢, 文件体积也更大。
    :type indent_format: "布尔值"

    :param overwrite: 默认 ``False``, 当为``True``时, 如果写入路径已经存在, 则会
      自动覆盖原文件。而为``False``时, 则会打印警告文件, 防止误操作覆盖源文件。
    :type overwrite: "布尔值"

    :param float_precision: 默认 ``None``, 当为任意整数时, 则会保留小数点后N位
    :type float_precision: "整数"

    :param enable_verbose: 默认 ``True``, 信息提示的开关, 批处理时建议关闭
    :type enable_verbose: ``布尔值``
    """
    prt("\nDump to '%s' ..." % abspath, enable_verbose)

    abspath = lower_ext(str(abspath))
    is_json = is_json_file(abspath)

    if os.path.exists(abspath):
        if not overwrite:  # 存在, 并且overwrite=False
            prt("    Stop! File exists and overwrite is not allowed",
                enable_verbose)
            return

    if float_precision is not None:
        encoder.FLOAT_REPR = lambda x: format(x, ".%sf" % float_precision)
        indent_format = True
    else:
        encoder.FLOAT_REPR = repr

    if indent_format:
        sort_keys = True
        indent = 4
    else:
        sort_keys = False
        indent = None

    st = time.clock()
    js = json.dumps(data, sort_keys=sort_keys, indent=indent,
                    ensure_ascii=ensure_ascii)
    content = js.encode("utf-8")
    if is_json:
        textfile.writebytes(content, abspath)
    else:
        compress.write_gzip(content, abspath)

    prt("    Complete! Elapse %.6f sec." % (time.clock() - st), enable_verbose)


Instruction:
Here you have a function in Python 3, explain what it does
def safe_dump(data, abspath,
              indent_format=False,
              float_precision=None,
              ensure_ascii=True,
              enable_verbose=True):
    """A stable version of :func:`dump`, this method will silently overwrite
    existing file.

    There's a issue with :func:`dump`: If your program is interrupted while
    writing, you got an incomplete file, and you also lose the original file.
    So this method write json to a temporary file first, then rename to what
    you expect, and silently overwrite old one. This way can guarantee atomic
    write.

    **中文文档**

    在对文件进行写入时, 如果程序中断, 则会留下一个不完整的文件。如果使用了覆盖式
    写入, 则我们即没有得到新文件, 同时也丢失了原文件。所以为了保证写操作的原子性
    (要么全部完成, 要么全部都不完成), 更好的方法是: 首先将文件写入一个临时文件中,
    完成后再讲文件重命名, 覆盖旧文件。这样即使中途程序被中断, 也仅仅是留下了一个
    未完成的临时文件而已, 不会影响原文件。
    """
    abspath = lower_ext(str(abspath))
    abspath_temp = "%s.tmp" % abspath
    dump(data, abspath_temp,
         indent_format=indent_format, float_precision=float_precision,
         ensure_ascii=ensure_ascii,
         overwrite=True, enable_verbose=enable_verbose)
    shutil.move(abspath_temp, abspath)

Response: A stable version of dump that does not overwrite existing file.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def pretty_dumps(data):
    try:
        return json.dumps(data, sort_keys=True, indent=4, ensure_ascii=False)
    except:
        return json.dumps(data, sort_keys=True, indent=4, ensure_ascii=True)

Response: Return json string in pretty format.


Instruction:
Can you create a Python 3 function that
computes left and right padding values.

Response:
def _get_pad_left_right(small, large):
    """ Compute left and right padding values.

    Here we use the convention that if the padding
    size is odd, we pad the odd part to the right
    and the even part to the left.

    Parameters
    ----------
    small : int
        Old size of original 1D array
    large : int
        New size off padded 1D array

    Returns
    -------
    (padleft, padright) : tuple
        The proposed padding sizes.
    """
    assert small < large, "Can only pad when new size larger than old size"

    padsize = large - small
    if padsize % 2 != 0:
        leftpad = (padsize - 1)/2
    else:
        leftpad = padsize/2
    rightpad = padsize-leftpad

    return int(leftpad), int(rightpad)


Instruction:
Write a Python 3 script for
performing linear padding for complex array

Response:
def pad_add(av, size=None, stlen=10):
    """ Perform linear padding for complex array

    The input array `av` is padded with a linear ramp starting at the
    edges and going outwards to an average value computed from a band
    of thickness `stlen` at the outer boundary of the array.

    Pads will only be appended, not prepended to the array.

    If the input array is complex, pads will be complex numbers
    The average is computed for phase and amplitude separately.

    Parameters
    ----------
    av : complex 1D or 2D ndarray
        The array that will be padded.
    size : int or tuple of length 1 (1D) or tuple of length 2 (2D), optional
        The final size of the padded array. Defaults to double the size
        of the input array.
    stlen : int, optional
        The thickness of the frame within `av` that will be used to
        compute an average value for padding.

    Returns
    -------
    pv : complex 1D or 2D ndarray
        Padded array `av` with pads appended to right and bottom.
    """
    if size is None:
        size = list()
        for s in av.shape:
            size.append(int(2*s))
    elif not hasattr(size, "__len__"):
        size = [size]

    assert len(av.shape) in [1, 2], "Only 1D and 2D arrays!"
    assert len(av.shape) == len(
        size), "`size` must have same length as `av.shape`!"

    if len(av.shape) == 2:
        return _pad_add_2d(av, size, stlen)
    else:
        return _pad_add_1d(av, size, stlen)


Instruction:
How would you explain what the following Python 3 function does
def _pad_add_1d(av, size, stlen):
    assert len(size) == 1

    padx = _get_pad_left_right(av.shape[0], size[0])

    mask = np.zeros(av.shape, dtype=bool)
    mask[stlen:-stlen] = True
    border = av[~mask]
    if av.dtype.name.count("complex"):
        padval = np.average(np.abs(border)) * \
            np.exp(1j*np.average(np.angle(border)))
    else:
        padval = np.average(border)
    if np.__version__[:3] in ["1.7", "1.8", "1.9"]:
        end_values = ((padval, padval),)
    else:
        end_values = (padval,)
    bv = np.pad(av,
                padx,
                mode="linear_ramp",
                end_values=end_values)
    # roll the array so that the padding values are on the right
    bv = np.roll(bv, -padx[0], 0)
    return bv

Response: 1D component of pad_add


Instruction:
How would you code a function in Python 3 to
remove linear padding from array This is a convenience function that does the opposite of `pad_add`. Parameters ---------- pv : 1D or 2D ndarray The array from which the padding will be removed. size : tuple of length 1 (1D) or 2 (2D), optional The final size of the un-padded array. Defaults to half the size of the input array. Returns ------- pv : 1D or 2D ndarray Padded array `av` with pads appended to right and bottom.

Response:
def pad_rem(pv, size=None):
    """ Removes linear padding from array

    This is a convenience function that does the opposite
    of `pad_add`.

    Parameters
    ----------
    pv : 1D or 2D ndarray
        The array from which the padding will be removed.
    size : tuple of length 1 (1D) or 2 (2D), optional
        The final size of the un-padded array. Defaults to half the size
        of the input array.

    Returns
    -------
    pv : 1D or 2D ndarray
        Padded array `av` with pads appended to right and bottom.
    """
    if size is None:
        size = list()
        for s in pv.shape:
            assert s % 2 == 0, "Uneven size; specify correct size of output!"
            size.append(int(s/2))
    elif not hasattr(size, "__len__"):
        size = [size]

    assert len(pv.shape) in [1, 2], "Only 1D and 2D arrays!"
    assert len(pv.shape) == len(
        size), "`size` must have same length as `av.shape`!"

    if len(pv.shape) == 2:
        return pv[:size[0], :size[1]]
    else:
        return pv[:size[0]]


Instruction:
Given the following Python 3 function, write the documentation
def scan_file(self, filename, apikey):
        url = self.base_url + "file/scan"
        params = {'apikey': apikey}
        scanfile = {"file": open(filename, 'rb')}
        response = requests.post(url, files=scanfile, params=params)
        rate_limit_clear = self.rate_limit()
        if rate_limit_clear:
            if response.status_code == self.HTTP_OK:
                json_response = response.json()
                return json_response
            elif response.status_code == self.HTTP_RATE_EXCEEDED:
                time.sleep(20)
            else:
                self.logger.error("sent: %s, HTTP: %d", filename, response.status_code)

Response: Send a file to virus total for assessment


Instruction:
How would you explain what the following Python 3 function does
def binary_report(self, sha256sum, apikey):
        url = self.base_url + "file/report"
        params = {"apikey": apikey, "resource": sha256sum}
        rate_limit_clear = self.rate_limit()

        if rate_limit_clear:
            response = requests.post(url, data=params)

            if response.status_code == self.HTTP_OK:
                json_response = response.json()
                response_code = json_response['response_code']
                return json_response
            elif response.status_code == self.HTTP_RATE_EXCEEDED:
                time.sleep(20)
            else:
                self.logger.warning("retrieve report: %s, HTTP code: %d", os.path.basename(filename), response.status_code)

Response: retrieve report from file scan


Instruction:
Write a Python 3 script to
send IP address for list of past malicous domain associations

Response:
def send_ip(self, ipaddr, apikey):
        """
        Send IP address for list of past malicous domain associations
        """
        url = self.base_url + "ip-address/report"
        parameters = {"ip": ipaddr, "apikey": apikey}
        rate_limit_clear = self.rate_limit()
        if rate_limit_clear:
            response = requests.get(url, params=parameters)
            if response.status_code == self.HTTP_OK:
                json_response = response.json()
                return json_response
            elif response.status_code == self.HTTP_RATE_EXCEEDED:
                    time.sleep(20)
            else:
                self.logger.error("sent: %s, HTTP: %d", ipaddr, response.status_code)
            time.sleep(self.public_api_sleep_time)


Instruction:
Implement a Python 3 function for
sending URLS for list of past malicous associations

Response:
def url_report(self, scan_url, apikey):
        """
        Send URLS for list of past malicous associations
        """
        url = self.base_url + "url/report"
        params = {"apikey": apikey, 'resource': scan_url}
        rate_limit_clear = self.rate_limit()
        if rate_limit_clear:
            response = requests.post(url, params=params, headers=self.headers)
            if response.status_code == self.HTTP_OK:
                json_response = response.json()
                return json_response
            elif response.status_code == self.HTTP_RATE_EXCEEDED:
                time.sleep(20)
            else:
                self.logger.error("sent: %s, HTTP: %d", scan_url, response.status_code)
            time.sleep(self.public_api_sleep_time)


Instruction:
Implement a Python 3 function for
reading the requirements file and returns a list of packages that are hard and extra packages.

Response:
def _read_requirements(filename, extra_packages):
    """Returns a list of package requirements read from the file."""
    requirements_file = open(filename).read()

    hard_requirements = []
    for line in requirements_file.splitlines():
        if _is_requirement(line):
            if line.find(';') > -1:
                dep, condition = tuple(line.split(';'))
                extra_packages[condition.strip()].append(dep.strip())
            else:
                hard_requirements.append(line.strip())
    return hard_requirements, extra_packages


Instruction:
Write a Python 3 function for
returning the value of a field in a string separated by a delimiter

Response:
def field(ctx, text, index, delimiter=' '):
    """
    Reference a field in string separated by a delimiter
    """
    splits = text.split(delimiter)

    # remove our delimiters and whitespace
    splits = [f for f in splits if f != delimiter and len(f.strip()) > 0]

    index = conversions.to_integer(index, ctx)
    if index < 1:
        raise ValueError('Field index cannot be less than 1')

    if index <= len(splits):
        return splits[index-1]
    else:
        return ''


Instruction:
Can you write a function in Python 3 where it
converts the given date to the number of seconds since January 1st 1970 UTC

Response:
def epoch(ctx, datetime):
    """
    Converts the given date to the number of seconds since January 1st, 1970 UTC
    """
    return conversions.to_decimal(str(conversions.to_datetime(datetime, ctx).timestamp()), ctx)


Instruction:
Write a Python 3 function that can
read digits from text into TTS

Response:
def read_digits(ctx, text):
    """
    Formats digits in text for reading in TTS
    """
    def chunk(value, chunk_size):
        return [value[i: i + chunk_size] for i in range(0, len(value), chunk_size)]

    text = conversions.to_string(text, ctx).strip()
    if not text:
        return ''

    # trim off the plus for phone numbers
    if text[0] == '+':
        text = text[1:]

    length = len(text)

    # ssn
    if length == 9:
        result = ' '.join(text[:3])
        result += ' , ' + ' '.join(text[3:5])
        result += ' , ' + ' '.join(text[5:])
        return result

    # triplets, most international phone numbers
    if length % 3 == 0 and length > 3:
        chunks = chunk(text, 3)
        return ' '.join(','.join(chunks))

    # quads, credit cards
    if length % 4 == 0:
        chunks = chunk(text, 4)
        return ' '.join(','.join(chunks))

    # otherwise, just put a comma between each number
    return ','.join(text)


Instruction:
Explain what the following Python 3 code does
def remove_first_word(ctx, text):
    text = conversions.to_string(text, ctx).lstrip()
    first = first_word(ctx, text)
    return text[len(first):].lstrip() if first else ''

Response: Removes the first word from the given text string


Instruction:
How would you explain what the following Python 3 function does
def word(ctx, text, number, by_spaces=False):
    return word_slice(ctx, text, number, conversions.to_integer(number, ctx) + 1, by_spaces)

Response: Extracts the nth word from the given text string


Instruction:
Given the following Python 3 function, write the documentation
def word_count(ctx, text, by_spaces=False):
    text = conversions.to_string(text, ctx)
    by_spaces = conversions.to_boolean(by_spaces, ctx)
    return len(__get_words(text, by_spaces))

Response: Returns the number of words in the given text


Instruction:
Can you tell what is the following Python 3 function doing
def word_slice(ctx, text, start, stop=0, by_spaces=False):
    text = conversions.to_string(text, ctx)
    start = conversions.to_integer(start, ctx)
    stop = conversions.to_integer(stop, ctx)
    by_spaces = conversions.to_boolean(by_spaces, ctx)

    if start == 0:
        raise ValueError("Start word cannot be zero")
    elif start > 0:
        start -= 1  # convert to a zero-based offset

    if stop == 0:  # zero is treated as no end
        stop = None
    elif stop > 0:
        stop -= 1  # convert to a zero-based offset

    words = __get_words(text, by_spaces)

    selection = operator.getitem(words, slice(start, stop))

    # re-combine selected words with a single space
    return ' '.join(selection)

Response: Extracts a substring spanning from start to stop.


Instruction:
Can you generate the documentation for the following Python 3 function
def format_date(ctx, text):
    dt = conversions.to_datetime(text, ctx)
    return dt.astimezone(ctx.timezone).strftime(ctx.get_date_format(True))

Response: Takes a single parameter as string and returns it in the format defined by the org


Instruction:
Explain what the following Python 3 code does
def format_location(ctx, text):
    text = conversions.to_string(text, ctx)
    return text.split(">")[-1].strip()

Response: Takes a single parameter and returns the name of the leaf boundary


Instruction:
Here you have a function in Python 3, explain what it does
def regex_group(ctx, text, pattern, group_num):
    text = conversions.to_string(text, ctx)
    pattern = conversions.to_string(pattern, ctx)
    group_num = conversions.to_integer(group_num, ctx)

    expression = regex.compile(pattern, regex.UNICODE | regex.IGNORECASE | regex.MULTILINE | regex.V0)
    match = expression.search(text)

    if not match:
        return ""

    if group_num < 0 or group_num > len(match.groups()):
        raise ValueError("No such matching group %d" % group_num)

    return match.group(group_num)

Response: Tries to match the text with the given pattern and returns the value of matching group_num


Instruction:
How would you explain what the following Python 3 function does
def decimal_round(number, num_digits, rounding=ROUND_HALF_UP):
    exp = Decimal(10) ** -num_digits

    if num_digits >= 0:
        return number.quantize(exp, rounding)
    else:
        return exp * (number / exp).to_integral_value(rounding)

Response: Round a number to num_digits decimal places.


Instruction:
How would you implement a function in Python 3 that
parses an ISO8601 formatted datetime from a string value

Response:
def parse_json_date(value):
    """
    Parses an ISO8601 formatted datetime from a string value
    """
    if not value:
        return None

    return datetime.datetime.strptime(value, JSON_DATETIME_FORMAT).replace(tzinfo=pytz.UTC)


Instruction:
Write a Python 3 script to
format a datetime as ISO8601 in UTC with millisecond precision e. g. 2004 - 10 - 03T09 : 41.790Z

Response:
def format_json_date(value):
    """
    Formats a datetime as ISO8601 in UTC with millisecond precision, e.g. "2014-10-03T09:41:12.790Z"
    """
    if not value:
        return None

    # %f will include 6 microsecond digits
    micro_precision = value.astimezone(pytz.UTC).strftime(JSON_DATETIME_FORMAT)

    # only keep the milliseconds portion of the second fraction
    return micro_precision[:-4] + 'Z'


Instruction:
How would you implement a function in Python 3 that
removes all non - printable characters from a text string

Response:
def clean(ctx, text):
    """
    Removes all non-printable characters from a text string
    """
    text = conversions.to_string(text, ctx)
    return ''.join([c for c in text if ord(c) >= 32])


Instruction:
Can you write a function in Python 3 where it
joins the given text strings into one text string

Response:
def concatenate(ctx, *text):
    """
    Joins text strings into one text string
    """
    result = ''
    for arg in text:
        result += conversions.to_string(arg, ctx)
    return result


Instruction:
Here you have a function in Python 3, explain what it does
def fixed(ctx, number, decimals=2, no_commas=False):
    value = _round(ctx, number, decimals)
    format_str = '{:f}' if no_commas else '{:,f}'
    return format_str.format(value)

Response: Formats the given number in decimal format using a period and commas


Instruction:
Can you tell what is the following Python 3 function doing
def left(ctx, text, num_chars):
    num_chars = conversions.to_integer(num_chars, ctx)
    if num_chars < 0:
        raise ValueError("Number of chars can't be negative")
    return conversions.to_string(text, ctx)[0:num_chars]

Response: Returns the first num_chars characters in a text string


Instruction:
Given the following Python 3 function, write the documentation
def rept(ctx, text, number_times):
    if number_times < 0:
        raise ValueError("Number of times can't be negative")
    return conversions.to_string(text, ctx) * conversions.to_integer(number_times, ctx)

Response: Repeats text a given number of times


Instruction:
How would you explain what the following Python 3 function does
def right(ctx, text, num_chars):
    num_chars = conversions.to_integer(num_chars, ctx)
    if num_chars < 0:
        raise ValueError("Number of chars can't be negative")
    elif num_chars == 0:
        return ''
    else:
        return conversions.to_string(text, ctx)[-num_chars:]

Response: Returns the last num_chars characters in a text string


Instruction:
Explain what the following Python 3 code does
def substitute(ctx, text, old_text, new_text, instance_num=-1):
    text = conversions.to_string(text, ctx)
    old_text = conversions.to_string(old_text, ctx)
    new_text = conversions.to_string(new_text, ctx)

    if instance_num < 0:
        return text.replace(old_text, new_text)
    else:
        splits = text.split(old_text)
        output = splits[0]
        instance = 1
        for split in splits[1:]:
            sep = new_text if instance == instance_num else old_text
            output += sep + split
            instance += 1
        return output

Response: Substitutes new_text for old_text in a text string


Instruction:
Can you generate the documentation for the following Python 3 function
def _unicode(ctx, text):
    text = conversions.to_string(text, ctx)
    if len(text) == 0:
        raise ValueError("Text can't be empty")
    return ord(text[0])

Response: Returns a numeric code for the first character in a text string


Instruction:
Can you create a Python 3 function that
returns a date value

Response:
def date(ctx, year, month, day):
    """
    Defines a date value
    """
    return _date(conversions.to_integer(year, ctx), conversions.to_integer(month, ctx), conversions.to_integer(day, ctx))


Instruction:
How would you code a function in Python 3 to
calculate the number of days months or years between two dates.

Response:
def datedif(ctx, start_date, end_date, unit):
    """
    Calculates the number of days, months, or years between two dates.
    """
    start_date = conversions.to_date(start_date, ctx)
    end_date = conversions.to_date(end_date, ctx)
    unit = conversions.to_string(unit, ctx).lower()

    if start_date > end_date:
        raise ValueError("Start date cannot be after end date")

    if unit == 'y':
        return relativedelta(end_date, start_date).years
    elif unit == 'm':
        delta = relativedelta(end_date, start_date)
        return 12 * delta.years + delta.months
    elif unit == 'd':
        return (end_date - start_date).days
    elif unit == 'md':
        return relativedelta(end_date, start_date).days
    elif unit == 'ym':
        return relativedelta(end_date, start_date).months
    elif unit == 'yd':
        return (end_date - start_date.replace(year=end_date.year)).days

    raise ValueError("Invalid unit value: %s" % unit)


Instruction:
Write a Python 3 function that can
move a date by the given number of months

Response:
def edate(ctx, date, months):
    """
    Moves a date by the given number of months
    """
    return conversions.to_date_or_datetime(date, ctx) + relativedelta(months=conversions.to_integer(months, ctx))


Instruction:
Write a Python 3 function for
returning a time value

Response:
def time(ctx, hours, minutes, seconds):
    """
    Defines a time value
    """
    return _time(conversions.to_integer(hours, ctx), conversions.to_integer(minutes, ctx), conversions.to_integer(seconds, ctx))


Instruction:
Write a Python 3 function that can
return the absolute value of a number

Response:
def _abs(ctx, number):
    """
    Returns the absolute value of a number
    """
    return conversions.to_decimal(abs(conversions.to_decimal(number, ctx)), ctx)


Instruction:
Can you create a Python 3 function that
rounds a number down to the nearest integer

Response:
def _int(ctx, number):
    """
    Rounds a number down to the nearest integer
    """
    return conversions.to_integer(conversions.to_decimal(number, ctx).to_integral_value(ROUND_FLOOR), ctx)


Instruction:
Write a Python 3 script for
returning the maximum value of all arguments

Response:
def _max(ctx, *number):
    """
    Returns the maximum value of all arguments
    """
    if len(number) == 0:
        raise ValueError("Wrong number of arguments")

    result = conversions.to_decimal(number[0], ctx)
    for arg in number[1:]:
        arg = conversions.to_decimal(arg, ctx)
        if arg > result:
            result = arg
    return result


Instruction:
Explain what the following Python 3 code does
def mod(ctx, number, divisor):
    number = conversions.to_decimal(number, ctx)
    divisor = conversions.to_decimal(divisor, ctx)
    return number - divisor * _int(ctx, number / divisor)

Response: Returns the remainder after number is divided by divisor


Instruction:
Write a Python 3 function for
returning the result of a number raised to a power

Response:
def _power(ctx, number, power):
    """
    Returns the result of a number raised to a power
    """
    return decimal_pow(conversions.to_decimal(number, ctx), conversions.to_decimal(power, ctx))


Instruction:
Write a Python 3 script to
return a random integer number between the numbers you specify

Response:
def randbetween(ctx, bottom, top):
    """
    Returns a random integer number between the numbers you specify
    """
    bottom = conversions.to_integer(bottom, ctx)
    top = conversions.to_integer(top, ctx)
    return random.randint(bottom, top)


Instruction:
Make a summary of the following Python 3 code
def _round(ctx, number, num_digits):
    number = conversions.to_decimal(number, ctx)
    num_digits = conversions.to_integer(num_digits, ctx)

    return decimal_round(number, num_digits, ROUND_HALF_UP)

Response: Rounds a number to a specified number of digits


Instruction:
Given the following Python 3 function, write the documentation
def rounddown(ctx, number, num_digits):
    number = conversions.to_decimal(number, ctx)
    num_digits = conversions.to_integer(num_digits, ctx)

    return decimal_round(number, num_digits, ROUND_DOWN)

Response: Rounds a number down toward zero


Instruction:
Can you implement a function in Python 3 that
rounds a number up away from zero

Response:
def roundup(ctx, number, num_digits):
    """
    Rounds a number up, away from zero
    """
    number = conversions.to_decimal(number, ctx)
    num_digits = conversions.to_integer(num_digits, ctx)

    return decimal_round(number, num_digits, ROUND_UP)


Instruction:
Make a summary of the following Python 3 code
def _sum(ctx, *number):
    if len(number) == 0:
        raise ValueError("Wrong number of arguments")

    result = Decimal(0)
    for arg in number:
        result += conversions.to_decimal(arg, ctx)
    return result

Response: Returns the sum of all arguments


Instruction:
Can you implement a function in Python 3 that
truncates a number to an integer by removing the fractional part of the number

Response:
def trunc(ctx, number):
    """
    Truncates a number to an integer by removing the fractional part of the number
    """
    return conversions.to_integer(conversions.to_decimal(number, ctx).to_integral_value(ROUND_DOWN), ctx)


Instruction:
Write a Python 3 function for
returning TRUE if all its arguments evaluate to TRUE

Response:
def _and(ctx, *logical):
    """
    Returns TRUE if and only if all its arguments evaluate to TRUE
    """
    for arg in logical:
        if not conversions.to_boolean(arg, ctx):
            return False
    return True


Instruction:
Given the following Python 3 function, write the documentation
def _if(ctx, logical_test, value_if_true=0, value_if_false=False):
    return value_if_true if conversions.to_boolean(logical_test, ctx) else value_if_false

Response: Returns one value if the condition evaluates to TRUE and another value if the condition evaluates to FALSE.


Instruction:
Can you implement a function in Python 3 that
returns TRUE if any argument is TRUE

Response:
def _or(ctx, *logical):
    """
    Returns TRUE if any argument is TRUE
    """
    for arg in logical:
        if conversions.to_boolean(arg, ctx):
            return True
    return False


Instruction:
Can you generate the documentation for the following Python 3 function
def compute_sum_values(i, j, data1, data2):
    sum1_ij = 1.
    for idx, d in zip([i,j], [data1, data2]):
        if isinstance(d, field): sum1_ij *= d.wvalue[idx]
        elif isinstance(d, points): sum1_ij *= d.weights[idx]
        else:
            raise NotImplementedError("data type not recognized")
    sum2_ij = data1.weights[i] * data2.weights[j]

    return sum1_ij, sum2_ij

Response: Compute the sum1_ij and sum2_ij values given the input indices and data instances.


Instruction:
Can you implement a function in Python 3 that
sets up the internal object variables for the internal object.

Response:
def _setup(self):
        """
        Set the binning info we need from the `edges`
        """
        dtype = [('inv', 'f8'), ('min', 'f8'), ('max', 'f8'),('N', 'i4'), ('spacing','object')]
        dtype        = numpy.dtype(dtype)
        self._info   = numpy.empty(self.Ndim, dtype=dtype)
        self.min     = self._info['min']
        self.max     = self._info['max']
        self.N       = self._info['N']
        self.inv     = self._info['inv']
        self.spacing = self._info['spacing']

        for i, dim in enumerate(self.dims):

            self.N[i] = len(self.edges[i])-1
            self.min[i] = self.edges[i][0]
            self.max[i] = self.edges[i][-1]

            # determine the type of spacing
            self.spacing[i] = None
            lin_diff = numpy.diff(self.edges[i])
            with numpy.errstate(divide='ignore', invalid='ignore'):
                log_diff = numpy.diff(numpy.log10(self.edges[i]))
            if numpy.allclose(lin_diff, lin_diff[0]):
                self.spacing[i] = 'linspace'
                self.inv[i] = self.N[i] * 1.0 / (self.max[i] - self.min[i])
            elif numpy.allclose(log_diff, log_diff[0]):
                self.spacing[i] = 'logspace'
                self.inv[i] = self.N[i] * 1.0 / numpy.log10(self.max[i] / self.min[i])

        self.shape = self.N + 2

        # store Rmax
        self.Rmax = self.max[0]


Instruction:
How would you explain what the following Python 3 function does
def update_sums(self, r, i, j, data1, data2, sum1, sum2, N=None, centers_sum=None):
        # the summation values for this (r,i,j)
        sum1_ij, sum2_ij = compute_sum_values(i, j, data1, data2)

        # digitize
        digr = self.digitize(r, i, j, data1, data2)

        if len(digr) == 3 and isinstance(digr[1], dict):
            dig, paircoords, weights = digr
        elif len(digr) == 2 and isinstance(digr[1], dict):
            dig, paircoords = digr
            weights = None
        else:
            dig = digr
            paircoords = None
            weights = None

        # sum 1
        def add_one_channel(sum1c, sum1_ijc):
            if numpy.isscalar(sum1_ijc) or sum1_ijc.ndim == 1:
                sum1c.flat[:] += utils.bincount(dig, sum1_ijc, minlength=sum1c.size)
            else:
                for d in range(sum1c.shape[0]):
                    sum1c[d].flat[:] += utils.bincount(dig, sum1_ijc[...,d], minlength=sum1c[d].size)

        if self.channels:
            if weights is None:
                raise RuntimeError("`digitize` of multi channel paircount did not return a weight array for the channels")

            sum1_ij = weights * sum1_ij
            # sum1_ij[ichannel, dig, dim]
            for ichannel in range(len(self.channels)):
                add_one_channel(sum1[ichannel], sum1_ij[ichannel])
        else:
            # sum1_ij[dig, dim]
            add_one_channel(sum1, sum1_ij)

        # sum 2, if both data are not points
        if not numpy.isscalar(sum2):
            sum2.flat[:] += utils.bincount(dig, sum2_ij, minlength=sum2.size)

        if N is not None:
            if not paircoords:
                raise RuntimeError("Bin center is requested but not returned by digitize")
            # update the mean coords
            self._update_mean_coords(dig, N, centers_sum, **paircoords)

Response: This method is used to update the sum1 and sum2 values for the current record.