gabfon's picture
Upload folder using huggingface_hub
3e363e2 verified
Raw History Blame Contribute Delete
15.9 kB
"""Loaders for the second bounded coverage-expansion tier."""
from __future__ import annotations
import json
import tarfile
from pathlib import Path
from xml.etree import ElementTree
from graph.graph import EdgeRecord
from graph.real.causal import _make_graph, _node_id
from graph.real.corpus import RealGraph, SourceMetadata, relation_phrase
def _text(element: ElementTree.Element | None) -> str:
return " ".join("".join(element.itertext()).split()) if element is not None else ""
def load_mesh_xml(
path: str | Path,
source: SourceMetadata,
*,
max_records: int | None = None,
) -> RealGraph:
"""Load MeSH descriptors, synonyms, and tree hierarchy."""
root = ElementTree.parse(path).getroot()
records = list(root.findall(".//DescriptorRecord"))
if max_records is not None:
records = records[:max_records]
nodes: dict[str, str] = {}
labels: dict[str, str] = {}
attributes: dict[str, dict[str, str]] = {}
tree_to_node: dict[str, str] = {}
edges: list[EdgeRecord] = []
for record in records:
descriptor_id = _text(record.find("DescriptorUI"))
name = _text(record.find("DescriptorName/String"))
if not descriptor_id or not name:
continue
node_id = f"mesh:{descriptor_id}"
nodes[node_id] = "mesh_descriptor"
labels[node_id] = name
synonyms = [
_text(term.find("String"))
for term in record.findall("ConceptList/Concept/TermList/Term")
]
attributes[node_id] = {
"descriptor_id": descriptor_id,
"scope_note": _text(record.find("ScopeNote"))[:4000],
"synonyms": ", ".join(dict.fromkeys(item for item in synonyms if item))[
:4000
],
}
for tree_number in record.findall("TreeNumberList/TreeNumber"):
tree_to_node[_text(tree_number)] = node_id
for record in records:
descriptor_id = _text(record.find("DescriptorUI"))
source_id = f"mesh:{descriptor_id}"
for tree_number in record.findall("TreeNumberList/TreeNumber"):
tree = _text(tree_number)
parent_tree = tree.rsplit(".", 1)[0] if "." in tree else ""
parent_id = tree_to_node.get(parent_tree)
if parent_id and parent_id != source_id:
edges.append(
EdgeRecord(
source_id,
"is_a",
parent_id,
"mesh_descriptor",
"mesh_descriptor",
relation_phrase("is_a"),
{"evidence_kind": "nlm_mesh_hierarchy"},
)
)
for related in record.findall("SeeRelatedDescriptorList/SeeRelatedDescriptor"):
target_id = f"mesh:{_text(related)}"
if target_id in nodes:
edges.append(
EdgeRecord(
source_id,
"associated_with",
target_id,
"mesh_descriptor",
"mesh_descriptor",
relation_phrase("associated_with"),
{"evidence_kind": "nlm_mesh_related_descriptor"},
)
)
return _make_graph(
source,
nodes,
edges,
labels=labels,
node_attributes=attributes,
relation_ids={"is_a", "associated_with"},
)
def _dmp_fields(line: str) -> list[str]:
# Every taxdump row ends in "\t|"; left on, it stops the last column from
# matching "scientific name".
return [field.strip() for field in line.rstrip().removesuffix("\t|").split("\t|\t")]
def load_ncbi_taxdump(
archive: str | Path,
source: SourceMetadata,
*,
max_nodes: int = 150_000,
) -> RealGraph:
"""Load a bounded NCBI Taxonomy parent/name graph."""
node_rows: dict[str, tuple[str, str]] = {}
name_rows: dict[str, str] = {}
with tarfile.open(archive, "r:gz") as handle:
nodes_member = next(
member
for member in handle.getmembers()
if member.name.rsplit("/", 1)[-1] == "nodes.dmp"
)
nodes_file = handle.extractfile(nodes_member)
if nodes_file is None:
raise ValueError("new_taxdump archive has no nodes.dmp")
for raw in nodes_file:
fields = _dmp_fields(raw.decode("utf-8", "replace"))
if len(fields) < 3:
continue
tax_id, parent_id, rank = fields[:3]
node_rows[tax_id] = (parent_id, rank)
if len(node_rows) >= max_nodes:
break
selected = set(node_rows)
names_member = next(
member
for member in handle.getmembers()
if member.name.rsplit("/", 1)[-1] == "names.dmp"
)
names_file = handle.extractfile(names_member)
if names_file is None:
raise ValueError("new_taxdump archive has no names.dmp")
for raw in names_file:
fields = _dmp_fields(raw.decode("utf-8", "replace"))
if len(fields) < 4 or fields[0] not in selected:
continue
if fields[3] == "scientific name" or fields[0] not in name_rows:
name_rows[fields[0]] = fields[1]
nodes = {f"taxon:{tax_id}": "taxon" for tax_id in node_rows}
labels = {f"taxon:{tax_id}": name_rows.get(tax_id, tax_id) for tax_id in node_rows}
attributes = {
f"taxon:{tax_id}": {"rank": rank, "tax_id": tax_id}
for tax_id, (_, rank) in node_rows.items()
}
edges: list[EdgeRecord] = []
for tax_id, (parent_id, rank) in node_rows.items():
if parent_id not in node_rows or parent_id == tax_id:
continue
edges.append(
EdgeRecord(
f"taxon:{tax_id}",
"is_a",
f"taxon:{parent_id}",
"taxon",
"taxon",
relation_phrase("is_a"),
{
"evidence_kind": "ncbi_taxonomy_hierarchy",
"rank": rank,
},
)
)
return _make_graph(
source,
nodes,
edges,
labels=labels,
node_attributes=attributes,
relation_ids={"is_a"},
)
def load_openalex_jsonl(
path: str | Path,
source: SourceMetadata,
*,
max_works: int | None = None,
) -> RealGraph:
"""Load bounded OpenAlex work metadata without abstracts/full text."""
nodes: dict[str, str] = {}
labels: dict[str, str] = {}
attributes: dict[str, dict[str, str]] = {}
edges: list[EdgeRecord] = []
relations = {"written_by", "has_topic", "published_in", "affiliated_with", "cites"}
with Path(path).open(encoding="utf-8") as handle:
for index, line in enumerate(handle):
if max_works is not None and index >= max_works:
break
work = json.loads(line)
work_key = str(work.get("id", f"work:{index}")).rsplit("/", 1)[-1]
work_id = f"openalex:work:{work_key}"
title = str(work.get("title") or work_id)
nodes[work_id] = "work"
labels[work_id] = title[:240]
attributes[work_id] = {
"publication_year": str(work.get("publication_year", "")),
"work_type": str(work.get("type", "")),
"cited_by_count": str(work.get("cited_by_count", "")),
}
for authorship in work.get("authorships", ()):
author = authorship.get("author") or {}
author_key = str(author.get("id", "")).rsplit("/", 1)[-1]
if not author_key:
continue
author_id = f"openalex:author:{author_key}"
nodes[author_id] = "author"
labels[author_id] = str(author.get("display_name", author_key))[:200]
edges.append(
EdgeRecord(
work_id,
"written_by",
author_id,
"work",
"author",
relation_phrase("written_by"),
{"evidence_kind": "openalex_metadata"},
)
)
for institution in authorship.get("institutions", ()):
institution_key = str(institution.get("id", "")).rsplit("/", 1)[-1]
if not institution_key:
continue
institution_id = f"openalex:institution:{institution_key}"
nodes[institution_id] = "institution"
labels[institution_id] = str(
institution.get("display_name", institution_key)
)[:200]
edges.append(
EdgeRecord(
author_id,
"affiliated_with",
institution_id,
"author",
"institution",
relation_phrase("affiliated_with"),
{"evidence_kind": "openalex_metadata"},
)
)
for topic in work.get("topics", ()):
topic_key = str(topic.get("id", "")).rsplit("/", 1)[-1]
if not topic_key:
continue
topic_id = f"openalex:topic:{topic_key}"
nodes[topic_id] = "topic"
labels[topic_id] = str(topic.get("display_name", topic_key))[:200]
edges.append(
EdgeRecord(
work_id,
"has_topic",
topic_id,
"work",
"topic",
relation_phrase("has_topic"),
{"evidence_kind": "openalex_metadata"},
)
)
for referenced in work.get("referenced_works", ())[:64]:
reference_key = str(referenced).rsplit("/", 1)[-1]
target_id = f"openalex:work:{reference_key}"
nodes.setdefault(target_id, "work")
labels.setdefault(target_id, reference_key)
edges.append(
EdgeRecord(
work_id,
"cites",
target_id,
"work",
"work",
relation_phrase("cites"),
{"evidence_kind": "openalex_metadata"},
)
)
return _make_graph(
source,
nodes,
edges,
labels=labels,
node_attributes=attributes,
relation_ids=relations,
)
def load_clinicaltrials_jsonl(
path: str | Path,
source: SourceMetadata,
*,
max_studies: int | None = None,
) -> RealGraph:
"""Load normalized ClinicalTrials.gov study metadata as a graph."""
nodes: dict[str, str] = {}
labels: dict[str, str] = {}
attributes: dict[str, dict[str, str]] = {}
edges: list[EdgeRecord] = []
relations = {
"studies_condition",
"tests_intervention",
"sponsored_by",
"has_outcome",
}
with Path(path).open(encoding="utf-8") as handle:
for index, line in enumerate(handle):
if max_studies is not None and index >= max_studies:
break
study = json.loads(line)
protocol = study.get("protocolSection", study)
identification = protocol.get("identificationModule", {})
study_id = str(identification.get("nctId", f"study-{index}"))
node_id = f"trial:{study_id}"
nodes[node_id] = "clinical_trial"
labels[node_id] = str(identification.get("briefTitle", study_id))[:240]
design = protocol.get("designModule", {})
attributes[node_id] = {
"phase": ",".join(str(value) for value in design.get("phases", ())),
"study_type": str(design.get("studyType", "")),
"enrollment": str(
(design.get("enrollmentInfo") or {}).get("count", "")
),
"evidence_kind": "registered_trial_metadata",
}
conditions = protocol.get("conditionsModule", {}).get("conditions", ())
interventions = protocol.get("armsInterventionsModule", {}).get(
"interventions", ()
)
sponsors = protocol.get("sponsorCollaboratorsModule", {})
for value in conditions:
target_id = _node_id("trial_condition", str(value))
nodes[target_id] = "condition"
labels[target_id] = str(value)[:200]
edges.append(
EdgeRecord(
node_id,
"studies_condition",
target_id,
"clinical_trial",
"condition",
relation_phrase("studies_condition"),
{"evidence_kind": "registered_trial_metadata"},
)
)
for intervention in interventions:
name = str(intervention.get("name", "")).strip()
if not name:
continue
target_id = _node_id("trial_intervention", name)
nodes[target_id] = "intervention"
labels[target_id] = name[:200]
edges.append(
EdgeRecord(
node_id,
"tests_intervention",
target_id,
"clinical_trial",
"intervention",
relation_phrase("tests_intervention"),
{"evidence_kind": "registered_trial_metadata"},
)
)
sponsor = sponsors.get("leadSponsor") or {}
sponsor_name = str(sponsor.get("name", "")).strip()
if sponsor_name:
target_id = _node_id("trial_sponsor", sponsor_name)
nodes[target_id] = "sponsor"
labels[target_id] = sponsor_name[:200]
edges.append(
EdgeRecord(
node_id,
"sponsored_by",
target_id,
"clinical_trial",
"sponsor",
relation_phrase("sponsored_by"),
{"evidence_kind": "registered_trial_metadata"},
)
)
outcomes = protocol.get("outcomesModule", {}).get("outcomes", ())
for outcome in outcomes[:16]:
name = str(outcome.get("measure", "")).strip()
if not name:
continue
target_id = _node_id("trial_outcome", name)
nodes[target_id] = "outcome_measure"
labels[target_id] = name[:240]
edges.append(
EdgeRecord(
node_id,
"has_outcome",
target_id,
"clinical_trial",
"outcome_measure",
relation_phrase("has_outcome"),
{"evidence_kind": "registered_trial_metadata"},
)
)
return _make_graph(
source,
nodes,
edges,
labels=labels,
node_attributes=attributes,
relation_ids=relations,
)