"""Loaders for the second bounded coverage-expansion tier.""" from __future__ import annotations import json import tarfile from pathlib import Path from xml.etree import ElementTree from graph.graph import EdgeRecord from graph.real.causal import _make_graph, _node_id from graph.real.corpus import RealGraph, SourceMetadata, relation_phrase def _text(element: ElementTree.Element | None) -> str: return " ".join("".join(element.itertext()).split()) if element is not None else "" def load_mesh_xml( path: str | Path, source: SourceMetadata, *, max_records: int | None = None, ) -> RealGraph: """Load MeSH descriptors, synonyms, and tree hierarchy.""" root = ElementTree.parse(path).getroot() records = list(root.findall(".//DescriptorRecord")) if max_records is not None: records = records[:max_records] nodes: dict[str, str] = {} labels: dict[str, str] = {} attributes: dict[str, dict[str, str]] = {} tree_to_node: dict[str, str] = {} edges: list[EdgeRecord] = [] for record in records: descriptor_id = _text(record.find("DescriptorUI")) name = _text(record.find("DescriptorName/String")) if not descriptor_id or not name: continue node_id = f"mesh:{descriptor_id}" nodes[node_id] = "mesh_descriptor" labels[node_id] = name synonyms = [ _text(term.find("String")) for term in record.findall("ConceptList/Concept/TermList/Term") ] attributes[node_id] = { "descriptor_id": descriptor_id, "scope_note": _text(record.find("ScopeNote"))[:4000], "synonyms": ", ".join(dict.fromkeys(item for item in synonyms if item))[ :4000 ], } for tree_number in record.findall("TreeNumberList/TreeNumber"): tree_to_node[_text(tree_number)] = node_id for record in records: descriptor_id = _text(record.find("DescriptorUI")) source_id = f"mesh:{descriptor_id}" for tree_number in record.findall("TreeNumberList/TreeNumber"): tree = _text(tree_number) parent_tree = tree.rsplit(".", 1)[0] if "." in tree else "" parent_id = tree_to_node.get(parent_tree) if parent_id and parent_id != source_id: edges.append( EdgeRecord( source_id, "is_a", parent_id, "mesh_descriptor", "mesh_descriptor", relation_phrase("is_a"), {"evidence_kind": "nlm_mesh_hierarchy"}, ) ) for related in record.findall("SeeRelatedDescriptorList/SeeRelatedDescriptor"): target_id = f"mesh:{_text(related)}" if target_id in nodes: edges.append( EdgeRecord( source_id, "associated_with", target_id, "mesh_descriptor", "mesh_descriptor", relation_phrase("associated_with"), {"evidence_kind": "nlm_mesh_related_descriptor"}, ) ) return _make_graph( source, nodes, edges, labels=labels, node_attributes=attributes, relation_ids={"is_a", "associated_with"}, ) def _dmp_fields(line: str) -> list[str]: # Every taxdump row ends in "\t|"; left on, it stops the last column from # matching "scientific name". return [field.strip() for field in line.rstrip().removesuffix("\t|").split("\t|\t")] def load_ncbi_taxdump( archive: str | Path, source: SourceMetadata, *, max_nodes: int = 150_000, ) -> RealGraph: """Load a bounded NCBI Taxonomy parent/name graph.""" node_rows: dict[str, tuple[str, str]] = {} name_rows: dict[str, str] = {} with tarfile.open(archive, "r:gz") as handle: nodes_member = next( member for member in handle.getmembers() if member.name.rsplit("/", 1)[-1] == "nodes.dmp" ) nodes_file = handle.extractfile(nodes_member) if nodes_file is None: raise ValueError("new_taxdump archive has no nodes.dmp") for raw in nodes_file: fields = _dmp_fields(raw.decode("utf-8", "replace")) if len(fields) < 3: continue tax_id, parent_id, rank = fields[:3] node_rows[tax_id] = (parent_id, rank) if len(node_rows) >= max_nodes: break selected = set(node_rows) names_member = next( member for member in handle.getmembers() if member.name.rsplit("/", 1)[-1] == "names.dmp" ) names_file = handle.extractfile(names_member) if names_file is None: raise ValueError("new_taxdump archive has no names.dmp") for raw in names_file: fields = _dmp_fields(raw.decode("utf-8", "replace")) if len(fields) < 4 or fields[0] not in selected: continue if fields[3] == "scientific name" or fields[0] not in name_rows: name_rows[fields[0]] = fields[1] nodes = {f"taxon:{tax_id}": "taxon" for tax_id in node_rows} labels = {f"taxon:{tax_id}": name_rows.get(tax_id, tax_id) for tax_id in node_rows} attributes = { f"taxon:{tax_id}": {"rank": rank, "tax_id": tax_id} for tax_id, (_, rank) in node_rows.items() } edges: list[EdgeRecord] = [] for tax_id, (parent_id, rank) in node_rows.items(): if parent_id not in node_rows or parent_id == tax_id: continue edges.append( EdgeRecord( f"taxon:{tax_id}", "is_a", f"taxon:{parent_id}", "taxon", "taxon", relation_phrase("is_a"), { "evidence_kind": "ncbi_taxonomy_hierarchy", "rank": rank, }, ) ) return _make_graph( source, nodes, edges, labels=labels, node_attributes=attributes, relation_ids={"is_a"}, ) def load_openalex_jsonl( path: str | Path, source: SourceMetadata, *, max_works: int | None = None, ) -> RealGraph: """Load bounded OpenAlex work metadata without abstracts/full text.""" nodes: dict[str, str] = {} labels: dict[str, str] = {} attributes: dict[str, dict[str, str]] = {} edges: list[EdgeRecord] = [] relations = {"written_by", "has_topic", "published_in", "affiliated_with", "cites"} with Path(path).open(encoding="utf-8") as handle: for index, line in enumerate(handle): if max_works is not None and index >= max_works: break work = json.loads(line) work_key = str(work.get("id", f"work:{index}")).rsplit("/", 1)[-1] work_id = f"openalex:work:{work_key}" title = str(work.get("title") or work_id) nodes[work_id] = "work" labels[work_id] = title[:240] attributes[work_id] = { "publication_year": str(work.get("publication_year", "")), "work_type": str(work.get("type", "")), "cited_by_count": str(work.get("cited_by_count", "")), } for authorship in work.get("authorships", ()): author = authorship.get("author") or {} author_key = str(author.get("id", "")).rsplit("/", 1)[-1] if not author_key: continue author_id = f"openalex:author:{author_key}" nodes[author_id] = "author" labels[author_id] = str(author.get("display_name", author_key))[:200] edges.append( EdgeRecord( work_id, "written_by", author_id, "work", "author", relation_phrase("written_by"), {"evidence_kind": "openalex_metadata"}, ) ) for institution in authorship.get("institutions", ()): institution_key = str(institution.get("id", "")).rsplit("/", 1)[-1] if not institution_key: continue institution_id = f"openalex:institution:{institution_key}" nodes[institution_id] = "institution" labels[institution_id] = str( institution.get("display_name", institution_key) )[:200] edges.append( EdgeRecord( author_id, "affiliated_with", institution_id, "author", "institution", relation_phrase("affiliated_with"), {"evidence_kind": "openalex_metadata"}, ) ) for topic in work.get("topics", ()): topic_key = str(topic.get("id", "")).rsplit("/", 1)[-1] if not topic_key: continue topic_id = f"openalex:topic:{topic_key}" nodes[topic_id] = "topic" labels[topic_id] = str(topic.get("display_name", topic_key))[:200] edges.append( EdgeRecord( work_id, "has_topic", topic_id, "work", "topic", relation_phrase("has_topic"), {"evidence_kind": "openalex_metadata"}, ) ) for referenced in work.get("referenced_works", ())[:64]: reference_key = str(referenced).rsplit("/", 1)[-1] target_id = f"openalex:work:{reference_key}" nodes.setdefault(target_id, "work") labels.setdefault(target_id, reference_key) edges.append( EdgeRecord( work_id, "cites", target_id, "work", "work", relation_phrase("cites"), {"evidence_kind": "openalex_metadata"}, ) ) return _make_graph( source, nodes, edges, labels=labels, node_attributes=attributes, relation_ids=relations, ) def load_clinicaltrials_jsonl( path: str | Path, source: SourceMetadata, *, max_studies: int | None = None, ) -> RealGraph: """Load normalized ClinicalTrials.gov study metadata as a graph.""" nodes: dict[str, str] = {} labels: dict[str, str] = {} attributes: dict[str, dict[str, str]] = {} edges: list[EdgeRecord] = [] relations = { "studies_condition", "tests_intervention", "sponsored_by", "has_outcome", } with Path(path).open(encoding="utf-8") as handle: for index, line in enumerate(handle): if max_studies is not None and index >= max_studies: break study = json.loads(line) protocol = study.get("protocolSection", study) identification = protocol.get("identificationModule", {}) study_id = str(identification.get("nctId", f"study-{index}")) node_id = f"trial:{study_id}" nodes[node_id] = "clinical_trial" labels[node_id] = str(identification.get("briefTitle", study_id))[:240] design = protocol.get("designModule", {}) attributes[node_id] = { "phase": ",".join(str(value) for value in design.get("phases", ())), "study_type": str(design.get("studyType", "")), "enrollment": str( (design.get("enrollmentInfo") or {}).get("count", "") ), "evidence_kind": "registered_trial_metadata", } conditions = protocol.get("conditionsModule", {}).get("conditions", ()) interventions = protocol.get("armsInterventionsModule", {}).get( "interventions", () ) sponsors = protocol.get("sponsorCollaboratorsModule", {}) for value in conditions: target_id = _node_id("trial_condition", str(value)) nodes[target_id] = "condition" labels[target_id] = str(value)[:200] edges.append( EdgeRecord( node_id, "studies_condition", target_id, "clinical_trial", "condition", relation_phrase("studies_condition"), {"evidence_kind": "registered_trial_metadata"}, ) ) for intervention in interventions: name = str(intervention.get("name", "")).strip() if not name: continue target_id = _node_id("trial_intervention", name) nodes[target_id] = "intervention" labels[target_id] = name[:200] edges.append( EdgeRecord( node_id, "tests_intervention", target_id, "clinical_trial", "intervention", relation_phrase("tests_intervention"), {"evidence_kind": "registered_trial_metadata"}, ) ) sponsor = sponsors.get("leadSponsor") or {} sponsor_name = str(sponsor.get("name", "")).strip() if sponsor_name: target_id = _node_id("trial_sponsor", sponsor_name) nodes[target_id] = "sponsor" labels[target_id] = sponsor_name[:200] edges.append( EdgeRecord( node_id, "sponsored_by", target_id, "clinical_trial", "sponsor", relation_phrase("sponsored_by"), {"evidence_kind": "registered_trial_metadata"}, ) ) outcomes = protocol.get("outcomesModule", {}).get("outcomes", ()) for outcome in outcomes[:16]: name = str(outcome.get("measure", "")).strip() if not name: continue target_id = _node_id("trial_outcome", name) nodes[target_id] = "outcome_measure" labels[target_id] = name[:240] edges.append( EdgeRecord( node_id, "has_outcome", target_id, "clinical_trial", "outcome_measure", relation_phrase("has_outcome"), {"evidence_kind": "registered_trial_metadata"}, ) ) return _make_graph( source, nodes, edges, labels=labels, node_attributes=attributes, relation_ids=relations, )